You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2016/06/11 14:03:35 UTC
incubator-s2graph git commit: [S2GRAPH-70]: Automate the process of
building a distribution package include following subtasks. - [S2GRAPH-75]:
Use an embedded database as the default metadata storage - [S2GRAPH-74]: Add
a script that starts/stops
Repository: incubator-s2graph
Updated Branches:
refs/heads/master ed6d36696 -> fabb212dd
[S2GRAPH-70]: Automate the process of building a distribution package
include following subtasks.
- [S2GRAPH-75]: Use an embedded database as the default metadata storage
- [S2GRAPH-74]: Add a script that starts/stops a local hbase server
- [S2GRAPH-73]: Add an SBT task and relevant scripts to construct a package for distribution
JIRA:
[S2GRAPH-70] https://issues.apache.org/jira/browse/S2GRAPH-70
Pull Request:
Closes #57
Authors:
Jong Wook Kim: jongwook@nyu.edu
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/fabb212d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/fabb212d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/fabb212d
Branch: refs/heads/master
Commit: fabb212dd69d47977987dc2b8ab43e2ae31808d4
Parents: ed6d366
Author: DO YUNG YOON <st...@apache.org>
Authored: Sat Jun 11 23:03:59 2016 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sat Jun 11 23:03:59 2016 +0900
----------------------------------------------------------------------
.gitignore | 1 -
CHANGES | 12 +
bin/hbase-standalone.sh | 41 ++++
bin/run-rat.sh | 31 ---
bin/s2graph-common.sh | 42 ++++
bin/s2graph-daemon.sh | 169 +++++++++++++
bin/s2graph-env.sh | 60 +++++
bin/s2graph.sh | 32 +++
bin/start-s2graph.sh | 27 +++
bin/stop-s2graph.sh | 27 +++
build.sbt | 5 +-
conf/application.conf | 22 ++
conf/hbase-site.xml | 26 ++
conf/logback.xml | 32 +++
dev/run-rat.sh | 31 +++
loader/build.sbt | 2 +-
project/Packager.scala | 98 ++++++++
s2core/build.sbt | 13 +-
s2core/lib/async-1.4.0.jar | Bin 18031 -> 0 bytes
s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar | Bin 1302410 -> 0 bytes
s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar | Bin 0 -> 5594923 bytes
s2core/lib/netty-3.9.4.Final.jar | Bin 1310154 -> 0 bytes
s2core/migrate/mysql/schema.sql | 238 -------------------
.../org/apache/s2graph/core/mysqls/schema.sql | 227 ++++++++++++++++++
.../org/apache/s2graph/core/mysqls/setup.sql | 28 +++
.../org/apache/s2graph/core/mysqls/Model.scala | 38 ++-
s2counter_core/build.sbt | 4 +
s2counter_loader/build.sbt | 1 +
s2rest_netty/build.sbt | 1 +
s2rest_netty/conf/reference.conf | 2 +-
s2rest_netty/src/main/resources/reference.conf | 2 +-
s2rest_play/build.sbt | 1 +
spark/build.sbt | 1 +
33 files changed, 931 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 1e21302..4b2e63d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,7 +90,6 @@ project/boot/
### PlayFramework ###
# Ignore Play! working directory #
-bin/
/db
.eclipse
/lib/
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 3aa2e7b..a7f7512 100644
--- a/CHANGES
+++ b/CHANGES
@@ -68,6 +68,9 @@ Release 0.12.1 - unreleased
S2GRAPH-22: Add missing shebang line to file.
(Contributed by Injun Song<ij...@gmail.com>, committed by DOYUNG YOON)
+
+ S2GRAPH-70: Automate the process of building a distribution package
+ (Contributed by Jong Wook Kim<jo...@nyu.edu>, committed by DOYUNG YOON)
BUG FIXES
@@ -118,6 +121,15 @@ Release 0.12.1 - unreleased
S2GRAPH-31: Remove playframework dependencies on s2core/build.sbt. (Committed by DOYUNG YOON).
S2GRAPH-87: LICENSE file should be changed. (Committed by Sergio Fern�ndez).
+
+ S2GRAPH-73: Add an SBT task and relevant scripts to construct a package for distribution.
+ (Contributed by Jong Wook Kim<jo...@nyu.edu>, committed by DOYUNG YOON)
+
+ S2GRAPH-74: Add a script that starts/stops a local hbase server.
+ (Contributed by Jong Wook Kim<jo...@nyu.edu>, committed by DOYUNG YOON)
+
+ S2GRAPH-75: Use an embedded database as the default metadata storage.
+ (Contributed by Jong Wook Kim<jo...@nyu.edu>, committed by DOYUNG YOON)
TEST
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/hbase-standalone.sh
----------------------------------------------------------------------
diff --git a/bin/hbase-standalone.sh b/bin/hbase-standalone.sh
new file mode 100755
index 0000000..9d9ef97
--- /dev/null
+++ b/bin/hbase-standalone.sh
@@ -0,0 +1,41 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# starts/stops/restarts an HBase server running in standalone mode.
+
+usage="Usage: hbase-standalone.sh (start|stop|restart|run|status)"
+
+bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
+
+# load environment variables
+. $bin/s2graph-env.sh
+. $bin/s2graph-common.sh
+
+# show usage when executed without enough arguments
+if [ $# -lt 1 ]; then
+ panic $usage
+fi
+
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dproc_master"
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.log.dir=$S2GRAPH_LOG_DIR"
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.log.file=hbase.logger.log"
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.home.dir=$S2GRAPH_HOME"
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.id.str=s2graph"
+S2GRAPH_OPTS="$S2GRAPH_OPTS -Dhbase.root.logger=INFO,RFA"
+
+export S2GRAPH_OPTS
+
+$bin/s2graph-daemon.sh $1 hbase start
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/run-rat.sh
----------------------------------------------------------------------
diff --git a/bin/run-rat.sh b/bin/run-rat.sh
deleted file mode 100644
index c22c938..0000000
--- a/bin/run-rat.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-base_dir=$(dirname $0)/..
-rat_excludes_file=$base_dir/.rat-excludes
-
-echo "Base Directory = $base_dir"
-
-if [ -z "$JAVA_HOME" ]; then
- JAVA="java"
-else
- JAVA="$JAVA_HOME/bin/java"
-fi
-
-rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.11.jar --dir $base_dir --exclude-file ${rat_excludes_file} -f -a"
-
-echo "Running " $rat_command
-$rat_command > $base_dir/rat.out
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-common.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph-common.sh b/bin/s2graph-common.sh
new file mode 100644
index 0000000..d03b22c
--- /dev/null
+++ b/bin/s2graph-common.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# print an error message to STDERR and quit
+panic() {
+ echo "$@" 1>&2
+ exit -1
+}
+
+# stop a process with specified pid, first by SIGTERM and then by SIGKILL
+terminate() {
+ for pid in "$@"; do
+ started=$(date +%s)
+ kill $pid > /dev/null 2>&1
+ printf "Waiting process $pid to finish" 1>&2
+ while kill -0 $pid > /dev/null 2>&1; do
+ sleep 0.5
+ printf "." 1>&2
+ if [ $(( $(date +%s) - $started )) -ge $S2GRAPH_STOP_TIMEOUT ]; then
+ break
+ fi
+ done
+ printf "\n" 1>&2
+ if kill -0 $pid > /dev/null 2>&1; then
+ echo "Force stopping $pid with a SIGKILL" 1>&2
+ kill -9 $pid > /dev/null 2>&1
+ fi
+ done
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-daemon.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph-daemon.sh b/bin/s2graph-daemon.sh
new file mode 100755
index 0000000..18c327b
--- /dev/null
+++ b/bin/s2graph-daemon.sh
@@ -0,0 +1,169 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# starts/stops/restarts a java application as a daemon process
+# also able to run the application directly using run
+
+usage="This script is intended to be used by other scripts in this directory."
+usage=$"$usage\n Please refer to start-s2graph.sh and stop-s2graph.sh"
+usage=$"$usage\n Usage: s2graph-daemon.sh (start|stop|restart|run|status)"
+usage="$usage (s2rest_play|s2rest_netty|...) <args...>"
+
+bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
+
+# load environment variables
+. $bin/s2graph-env.sh
+. $bin/s2graph-common.sh
+
+# show usage when executed without enough arguments
+if [ $# -le 1 ]; then
+ panic "$usage"
+fi
+
+# to enable the JVM application to use relative paths as well
+cd "$S2GRAPH_HOME"
+
+# arguments
+action=$1
+shift
+service=$1
+shift
+this="$bin/$(basename ${BASH_SOURCE-$0})"
+args="$@"
+
+# locate java
+if [ -z $JAVA_HOME ]; then
+ JAVA=$(which java)
+else
+ JAVA=$JAVA_HOME/bin/java
+fi
+
+# require JVM 8
+JAVA_VERSION=$($JAVA -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
+if [ $JAVA_VERSION -lt 18 ]; then
+ panic "Java 8 or later is required"
+fi
+
+# construct the classpath
+classpath="$S2GRAPH_CONF_DIR:$S2GRAPH_LIB_DIR/*"
+
+# determine the main class
+case $service in
+s2rest_play)
+ main="play.core.server.NettyServer"
+ ;;
+hbase)
+ main="org.apache.hadoop.hbase.master.HMaster"
+ ;;
+*)
+ panic "Unknown service: $service"
+esac
+
+# file that contains the service's pid
+pidfile="$S2GRAPH_PID_DIR/$service.pid"
+
+# file that contains the console output of the JVM
+# this file should only contain abnormal logs, while
+# the normal logs should go to the log files as configured separately,
+# ideally with daily log-rotation.
+log="$S2GRAPH_LOG_DIR/$service.log"
+
+start() {
+ if [ -f "$pidfile" ]; then
+ if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then
+ panic "$service is already running as process $(cat "$pidfile")"
+ fi
+ rm -r "$pidfile"
+ fi
+ echo "Starting $service..." 1>&2
+
+ "$this" run "$service" $args < /dev/null >> "$log" 2>&1 &
+ disown -h -r
+
+ # sleep enough, to check if there was any error during startup
+ sleep 3
+ if [ -f "$pidfile" ]; then
+ echo "Service $service is running as process $(cat "$pidfile")"
+ else
+ echo "Service $service might have failed to start; see $log"
+ fi
+}
+
+stop() {
+ if [ -f "$pidfile" ]; then
+ if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then
+ echo "Stopping $service" 1>&2
+ terminate $(cat "$pidfile")
+ else
+ echo "Process $(cat "$pidfile") is not running" 1>&2
+ fi
+ else
+ echo "PID file does not exist at $pidfile" 1>&2
+ fi
+ rm -f "$pidfile"
+}
+
+cleanup() {
+ if [ -f "$pidfile" ]; then
+ if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then
+ terminate $(cat "$pidfile")
+ fi
+ rm -f "$pidfile"
+ fi
+}
+
+status() {
+ if [ -f "$pidfile" ]; then
+ if kill -0 $(cat "$pidfile") > /dev/null 2>&1; then
+ echo "Service $service is running as process $(cat "$pidfile")" 1>&2
+ else
+ echo "PID file for service $service exists at $pidfile, " \
+ "but process $(cat "$pidfile") is not running" 1>&2
+ fi
+ else
+ echo "Service $service is not running" 1>&2
+ fi
+}
+
+run() {
+ trap cleanup SIGHUP SIGINT SIGTERM EXIT
+ echo "$JAVA" -cp "$classpath" -Ds2graph.home="$S2GRAPH_HOME" $S2GRAPH_OPTS "$main" $args 2>&1 &
+ "$JAVA" -cp "$classpath" -Ds2graph.home="$S2GRAPH_HOME" $S2GRAPH_OPTS "$main" $args 2>&1 &
+ pid=$!
+ echo "$pid" > "$pidfile"
+ wait "$pid"
+}
+
+case $action in
+start)
+ start
+ ;;
+stop)
+ stop
+ ;;
+restart)
+ stop
+ start
+ ;;
+status)
+ status
+ ;;
+run)
+ run
+ ;;
+*)
+ panic "$usage"
+esac
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph-env.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph-env.sh b/bin/s2graph-env.sh
new file mode 100644
index 0000000..0145e1d
--- /dev/null
+++ b/bin/s2graph-env.sh
@@ -0,0 +1,60 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file sets appropriate environment variables for running S2Graph.
+# Should be sourced and not be directly executed.
+# The executable flag for this file should not be set, e.g. 644.
+
+# Find the current directory
+SOURCE="${BASH_SOURCE[0]}"
+# resolve $SOURCE until the file is no longer a symlink
+while [ -h "$SOURCE" ]; do
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE"
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+# set S2GRAPH_HOME
+if [ -z $S2GRAPH_HOME ]; then
+ export S2GRAPH_HOME="$( cd -P "$DIR/.." && pwd )"
+fi
+
+# set other environment variables for directories, relative to $S2GRAPH_HOME
+# not intended to be individually configurable,
+# configurations can be added or overridden in conf/application.conf
+export S2GRAPH_BIN_DIR=$S2GRAPH_HOME/bin
+export S2GRAPH_LIB_DIR=$S2GRAPH_HOME/lib
+export S2GRAPH_LOG_DIR=$S2GRAPH_HOME/logs
+export S2GRAPH_CONF_DIR=$S2GRAPH_HOME/conf
+export S2GRAPH_HBASE_DIR=$S2GRAPH_HOME/var/hbase
+export S2GRAPH_METASTORE_DIR=$S2GRAPH_HOME/var/metastore
+export S2GRAPH_PID_DIR=$S2GRAPH_HOME/var/pid
+
+mkdir -p "$S2GRAPH_LOG_DIR" "$S2GRAPH_CONF_DIR" "$S2GRAPH_HBASE_DIR" \
+ "$S2GRAPH_METASTORE_DIR" "$S2GRAPH_PID_DIR"
+
+# apply the minimal JVM settings if none was given;
+# in production, this variable is expected to be given by a deployment system.
+if [ -z "$S2GRAPH_OPTS" ]; then
+ export S2GRAPH_OPTS="-Xms1g -Xmx1g"
+fi
+
+# the maximum wait time for a process to finish, before sending SIGKILL
+if [ -z "$S2GRAPH_STOP_TIMEOUT" ]; then
+ export S2GRAPH_STOP_TIMEOUT=60 # in seconds
+fi
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/s2graph.sh b/bin/s2graph.sh
new file mode 100755
index 0000000..58ddc28
--- /dev/null
+++ b/bin/s2graph.sh
@@ -0,0 +1,32 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# starts/stops/restarts an S2Graph server
+
+usage="Usage: s2graph.sh (start|stop|restart|run|status)"
+
+bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
+
+# load environment variables
+. $bin/s2graph-env.sh
+. $bin/s2graph-common.sh
+
+# show usage when executed without enough arguments
+if [ $# -lt 1 ]; then
+ panic $usage
+fi
+
+$bin/s2graph-daemon.sh $1 s2rest_play
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/start-s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/start-s2graph.sh b/bin/start-s2graph.sh
new file mode 100755
index 0000000..23ab1ab
--- /dev/null
+++ b/bin/start-s2graph.sh
@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Starts the S2Graph server, with an hbase server as required
+
+bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
+
+# load environment variables
+. $bin/s2graph-env.sh
+. $bin/s2graph-common.sh
+
+$bin/hbase-standalone.sh start
+$bin/s2graph.sh start
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/bin/stop-s2graph.sh
----------------------------------------------------------------------
diff --git a/bin/stop-s2graph.sh b/bin/stop-s2graph.sh
new file mode 100755
index 0000000..124c192
--- /dev/null
+++ b/bin/stop-s2graph.sh
@@ -0,0 +1,27 @@
+#! /usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Stops the S2Graph server, along with an hbase server as required
+
+bin=$(cd "$(dirname "${BASH_SOURCE-$0}")">/dev/null; pwd)
+
+# load environment variables
+. $bin/s2graph-env.sh
+. $bin/s2graph-common.sh
+
+$bin/s2graph.sh stop
+$bin/hbase-standalone.sh stop
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index 8571163..d3c6692 100755
--- a/build.sbt
+++ b/build.sbt
@@ -64,10 +64,13 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
lazy val root = (project in file("."))
.aggregate(s2core, s2rest_play)
+ .dependsOn(s2rest_play, s2rest_netty, loader, s2counter_loader) // this enables packaging on the root project
.settings(commonSettings: _*)
lazy val runRatTask = taskKey[Unit]("Runs Apache rat on S2Graph")
runRatTask := {
- "sh bin/run-rat.sh" !
+ "sh dev/run-rat.sh" !
}
+
+Packager.defaultSettings
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/application.conf
----------------------------------------------------------------------
diff --git a/conf/application.conf b/conf/application.conf
new file mode 100644
index 0000000..39b5f68
--- /dev/null
+++ b/conf/application.conf
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL"
+db.default.user="sa"
+db.default.password="sa"
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/hbase-site.xml
----------------------------------------------------------------------
diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
new file mode 100644
index 0000000..adf29db
--- /dev/null
+++ b/conf/hbase-site.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>${s2graph.home}/var/hbase</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/conf/logback.xml
----------------------------------------------------------------------
diff --git a/conf/logback.xml b/conf/logback.xml
new file mode 100644
index 0000000..d3f09bc
--- /dev/null
+++ b/conf/logback.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <layout class="ch.qos.logback.classic.PatternLayout">
+ <Pattern>
+ %d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
+ </Pattern>
+ </layout>
+ </appender>
+ <root level="INFO">
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/dev/run-rat.sh
----------------------------------------------------------------------
diff --git a/dev/run-rat.sh b/dev/run-rat.sh
new file mode 100644
index 0000000..c22c938
--- /dev/null
+++ b/dev/run-rat.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+base_dir=$(dirname $0)/..
+rat_excludes_file=$base_dir/.rat-excludes
+
+echo "Base Directory = $base_dir"
+
+if [ -z "$JAVA_HOME" ]; then
+ JAVA="java"
+else
+ JAVA="$JAVA_HOME/bin/java"
+fi
+
+rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.11.jar --dir $base_dir --exclude-file ${rat_excludes_file} -f -a"
+
+echo "Running " $rat_command
+$rat_command > $base_dir/rat.out
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/loader/build.sbt
----------------------------------------------------------------------
diff --git a/loader/build.sbt b/loader/build.sbt
index 2929da8..b74f305 100644
--- a/loader/build.sbt
+++ b/loader/build.sbt
@@ -28,6 +28,7 @@ projectDependencies := Seq(
)
libraryDependencies ++= Seq(
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-hive" % Common.sparkVersion % "provided",
@@ -35,7 +36,6 @@ libraryDependencies ++= Seq(
"org.apache.httpcomponents" % "fluent-hc" % "4.2.5",
"org.specs2" %% "specs2-core" % "2.4.11" % "test",
"org.scalatest" %% "scalatest" % "2.2.1" % "test",
- "org.apache.hbase" % "hbase-hadoop-compat" % "0.98.14-hadoop2",
"org.apache.hadoop" % "hadoop-distcp" % Common.hadoopVersion
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/project/Packager.scala
----------------------------------------------------------------------
diff --git a/project/Packager.scala b/project/Packager.scala
new file mode 100644
index 0000000..cff65fd
--- /dev/null
+++ b/project/Packager.scala
@@ -0,0 +1,98 @@
+import java.nio.file.attribute.BasicFileAttributes
+import java.nio.file.{FileVisitResult, Files, Path, Paths, SimpleFileVisitor, StandardCopyOption}
+
+import sbt.Keys._
+import sbt._
+
+object Packager {
+
+ val packagedJars = TaskKey[Seq[File]]("packaged-jars", "list of jar files for packaging")
+ val packager = TaskKey[File]("package", s"packages the project for deployment")
+
+ val packagedJarsTask = (state, thisProjectRef).flatMap { (state, project) =>
+
+ /** return all tasks of given TaskKey's dependencies */
+ def taskInAllDependencies[T](taskKey: TaskKey[T]): Task[Seq[T]] = {
+ val structure = Project.structure(state)
+
+ Dag.topologicalSort(project) { ref =>
+ Project.getProject(ref, structure).toList.flatMap { p =>
+ p.dependencies.map(_.project) ++ p.aggregate
+ }
+ }.flatMap { p =>
+ taskKey in p get structure.data
+ }.join
+ }
+
+ for {
+ packaged <- taskInAllDependencies(packagedArtifacts)
+ srcs <- taskInAllDependencies(packageSrc in Compile)
+ docs <- taskInAllDependencies(packageDoc in Compile)
+ } yield {
+ packaged.flatten
+ .collect { case (artifact, path) if artifact.extension == "jar" => path }
+ .diff(srcs ++ docs) //remove srcs & docs since we do not need them in the dist
+ .distinct
+ }
+ }
+
+ val packagerTask = (baseDirectory, packagedJars, dependencyClasspath in Runtime, target, streams).map {
+ (root, packaged, dependencies, target, streams) =>
+ val base = target / "deploy"
+
+ IO.delete(base)
+ IO.createDirectory(base)
+
+ val subdirectories = Seq("bin", "conf", "lib")
+
+ for (dir <- subdirectories) {
+ IO.createDirectory(base / dir)
+ }
+
+ // copy jars to /lib
+ val libs = dependencies.filter(_.data.ext == "jar").map(_.data) ++ packaged
+ libs.foreach { jar =>
+ val source = jar.toPath
+ val target = Paths.get(base.getPath, "lib", jar.getName)
+
+ try {
+ // try to create hard links for speed and to save disk space
+ Files.createLink(target, source)
+ streams.log.debug(s"created link: $source -> $target")
+ } catch {
+ case e: Any =>
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING)
+ streams.log.debug(s"copied: $source -> $target")
+ }
+ }
+
+ // copy other files recursively
+ for (subdir <- subdirectories if subdir != "lib") {
+ val source = (root / subdir).toPath
+ val target = (base / subdir).toPath
+ Files.walkFileTree(source, new SimpleFileVisitor[Path] {
+ override def preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult = {
+ val dest = target.resolve(source.relativize(dir))
+ Files.createDirectories(dest)
+ streams.log.debug(s"created directory: $dest")
+ FileVisitResult.CONTINUE
+ }
+
+ override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+ val dest = target.resolve(source.relativize(file))
+ Files.copy(file, dest)
+ streams.log.debug(s"copied: $file -> $dest")
+ FileVisitResult.CONTINUE
+ }
+ })
+ }
+
+ streams.log.info(s"package for distribution located at $base")
+ base
+ }
+
+ val defaultSettings = Seq(
+ packagedJars <<= packagedJarsTask,
+ packager <<= packagerTask
+ )
+}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/build.sbt
----------------------------------------------------------------------
diff --git a/s2core/build.sbt b/s2core/build.sbt
index 6774c60..46c2795 100644
--- a/s2core/build.sbt
+++ b/s2core/build.sbt
@@ -26,15 +26,18 @@ libraryDependencies ++= Seq(
"com.typesafe" % "config" % "1.2.1",
"com.typesafe.play" %% "play-json" % Common.playVersion,
"com.typesafe.akka" %% "akka-actor" % "2.3.4",
- "org.apache.hbase" % "hbase-client" % Common.hbaseVersion excludeAll ExclusionRule(organization = "org.slf4j"),
- "org.apache.hbase" % "hbase-common" % Common.hbaseVersion excludeAll ExclusionRule(organization = "org.slf4j"),
- "org.apache.hbase" % "hbase-server" % Common.hbaseVersion excludeAll(ExclusionRule(organization = "org.slf4j"), ExclusionRule(organization = "com.google.protobuf")),
- "org.apache.hadoop" % "hadoop-common" % Common.hadoopVersion excludeAll ExclusionRule(organization = "org.slf4j"),
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
+ "org.apache.hbase" % "hbase-client" % Common.hbaseVersion exclude("org.slf4j", "*"),
+ "org.apache.hbase" % "hbase-common" % Common.hbaseVersion exclude("org.slf4j", "*"),
+ "org.apache.hbase" % "hbase-server" % Common.hbaseVersion exclude("org.slf4j", "*") exclude("com.google.protobuf", "*"),
+ "org.apache.hbase" % "hbase-hadoop-compat" % Common.hbaseVersion exclude("org.slf4j", "*"),
+ "org.apache.hbase" % "hbase-hadoop2-compat" % Common.hbaseVersion exclude("org.slf4j", "*"),
+ "org.apache.kafka" % "kafka-clients" % "0.8.2.0" exclude("org.slf4j", "*") exclude("com.sun.jdmk", "*") exclude("com.sun.jmx", "*") exclude("javax.jms", "*"),
"commons-pool" % "commons-pool" % "1.6",
"org.scalatest" %% "scalatest" % "2.2.4" % "test",
"org.scalikejdbc" %% "scalikejdbc" % "2.1.+",
"mysql" % "mysql-connector-java" % "5.1.28",
- "org.apache.kafka" % "kafka-clients" % "0.8.2.0" excludeAll(ExclusionRule(organization = "org.slf4j"), ExclusionRule(organization = "com.sun.jdmk"), ExclusionRule(organization = "com.sun.jmx"), ExclusionRule(organization = "javax.jms")),
+ "com.h2database" % "h2" % "1.4.192",
"com.github.danielwegener" % "logback-kafka-appender" % "0.0.4"
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/async-1.4.0.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/async-1.4.0.jar b/s2core/lib/async-1.4.0.jar
deleted file mode 100644
index 7fa1692..0000000
Binary files a/s2core/lib/async-1.4.0.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar b/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar
deleted file mode 100644
index 801d87d..0000000
Binary files a/s2core/lib/asynchbase-1.7.1-SNAPSHOT.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar b/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar
new file mode 100644
index 0000000..ade9f82
Binary files /dev/null and b/s2core/lib/asynchbase-1.7.2-SNAPSHOT.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/lib/netty-3.9.4.Final.jar
----------------------------------------------------------------------
diff --git a/s2core/lib/netty-3.9.4.Final.jar b/s2core/lib/netty-3.9.4.Final.jar
deleted file mode 100644
index 7ee6ba4..0000000
Binary files a/s2core/lib/netty-3.9.4.Final.jar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/migrate/mysql/schema.sql
----------------------------------------------------------------------
diff --git a/s2core/migrate/mysql/schema.sql b/s2core/migrate/mysql/schema.sql
deleted file mode 100644
index 9ce746c..0000000
--- a/s2core/migrate/mysql/schema.sql
+++ /dev/null
@@ -1,238 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing,
--- software distributed under the License is distributed on an
--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- KIND, either express or implied. See the License for the
--- specific language governing permissions and limitations
--- under the License.
---
-
-CREATE DATABASE IF NOT EXISTS graph_dev;
-
-CREATE USER 'graph'@'%' IDENTIFIED BY 'graph';
-
-GRANT ALL PRIVILEGES ON graph_dev.* TO 'graph'@'%' identified by 'graph';
-
-flush privileges;
-
-use graph_dev;
-
-
-SET FOREIGN_KEY_CHECKS = 0;
-
--- ----------------------------
--- Table structure for `services`
--- ----------------------------
-DROP TABLE IF EXISTS `services`;
-CREATE TABLE `services` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `service_name` varchar(64) NOT NULL,
- `access_token` varchar(64) NOT NULL,
- `cluster` varchar(255) NOT NULL,
- `hbase_table_name` varchar(255) NOT NULL,
- `pre_split_size` integer NOT NULL default 0,
- `hbase_table_ttl` integer,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_service_name` (`service_name`),
- INDEX `idx_access_token` (`access_token`),
- INDEX `idx_cluster` (cluster(75))
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-
--- ----------------------------
--- Table structure for `services_columns`
--- ----------------------------
-DROP TABLE IF EXISTS `service_columns`;
-CREATE TABLE `service_columns` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `service_id` integer NOT NULL,
- `column_name` varchar(64) NOT NULL,
- `column_type` varchar(8) NOT NULL,
- `schema_version` varchar(8) NOT NULL default 'v2',
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-ALTER TABLE service_columns add FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE;
-
-
--- ----------------------------
--- Table structure for `column_metas`
--- ----------------------------
-DROP TABLE IF EXISTS `column_metas`;
-CREATE TABLE `column_metas` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `column_id` integer NOT NULL,
- `name` varchar(64) NOT NULL,
- `seq` tinyint NOT NULL,
- `data_type` varchar(8) NOT NULL DEFAULT 'string',
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_column_id_name` (`column_id`, `name`),
- INDEX `idx_column_id_seq` (`column_id`, `seq`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-ALTER TABLE column_metas ADD FOREIGN KEY(column_id) REFERENCES service_columns(id) ON DELETE CASCADE;
-
--- ----------------------------
--- Table structure for `labels`
--- ----------------------------
-
-DROP TABLE IF EXISTS `labels`;
-CREATE TABLE `labels` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `label` varchar(64) NOT NULL,
- `src_service_id` integer NOT NULL,
- `src_column_name` varchar(64) NOT NULL,
- `src_column_type` varchar(8) NOT NULL,
- `tgt_service_id` integer NOT NULL,
- `tgt_column_name` varchar(64) NOT NULL,
- `tgt_column_type` varchar(8) NOT NULL,
- `is_directed` tinyint NOT NULL DEFAULT 1,
- `service_name` varchar(64),
- `service_id` integer NOT NULL,
- `consistency_level` varchar(8) NOT NULL DEFAULT 'weak',
- `hbase_table_name` varchar(255) NOT NULL DEFAULT 's2graph',
- `hbase_table_ttl` integer,
- `schema_version` varchar(8) NOT NULL default 'v2',
- `is_async` tinyint(4) NOT NULL default '0',
- `compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4',
- `options` text,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_label` (`label`),
- INDEX `idx_src_column_name` (`src_column_name`),
- INDEX `idx_tgt_column_name` (`tgt_column_name`),
- INDEX `idx_src_service_id` (`src_service_id`),
- INDEX `idx_tgt_service_id` (`tgt_service_id`),
- INDEX `idx_service_name` (`service_name`),
- INDEX `idx_service_id` (`service_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id);
-
-
-
--- ----------------------------
--- Table structure for `label_metas`
--- ----------------------------
-DROP TABLE IF EXISTS `label_metas`;
-CREATE TABLE `label_metas` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `label_id` integer NOT NULL,
- `name` varchar(64) NOT NULL,
- `seq` tinyint NOT NULL,
- `default_value` varchar(64) NOT NULL,
- `data_type` varchar(8) NOT NULL DEFAULT 'long',
- `used_in_index` tinyint NOT NULL DEFAULT 0,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_label_id_name` (`label_id`, `name`),
- INDEX `idx_label_id_seq` (`label_id`, `seq`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-ALTER TABLE label_metas ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
-
-
--- ----------------------------
--- Table structure for `label_indices`
--- ----------------------------
-DROP TABLE IF EXISTS `label_indices`;
-CREATE TABLE `label_indices` (
- `id` int(11) NOT NULL AUTO_INCREMENT,
- `label_id` int(11) NOT NULL,
- `name` varchar(64) NOT NULL DEFAULT '_PK',
- `seq` tinyint(4) NOT NULL,
- `meta_seqs` varchar(64) NOT NULL,
- `formulars` varchar(255) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_label_id_seq` (`label_id`,`meta_seqs`),
- UNIQUE KEY `ux_label_id_name` (`label_id`,`name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
-
-
--- ----------------------------
--- Table structure for `experiments`
--- ----------------------------
-DROP TABLE IF EXISTS `experiments`;
-CREATE TABLE `experiments` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `service_id` integer NOT NULL,
- `service_name` varchar(128) NOT NULL,
- `name` varchar(64) NOT NULL,
- `description` varchar(255) NOT NULL,
- `experiment_type` varchar(8) NOT NULL DEFAULT 'u',
- `total_modular` int NOT NULL DEFAULT 100,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_service_id_name` (`service_id`, `name`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
--- ALTER TABLE experiments ADD FOREIGN KEY(service_id) REFERENCES service(id) ON DELETE CASCADE;
-
-
--- ----------------------------
--- Table structure for `buckets`
--- ----------------------------
-DROP TABLE IF EXISTS `buckets`;
-CREATE TABLE `buckets` (
- `id` integer NOT NULL AUTO_INCREMENT,
- `experiment_id` integer NOT NULL,
- `modular` varchar(64) NOT NULL,
- `http_verb` varchar(8) NOT NULL,
- `api_path` text NOT NULL,
- `uuid_key` varchar(128),
- `uuid_placeholder` varchar(64),
- `request_body` text NOT NULL,
- `timeout` int NOT NULL DEFAULT 1000,
- `impression_id` varchar(64) NOT NULL,
- `is_graph_query` tinyint NOT NULL DEFAULT 1,
- `is_empty` tinyint NOT NULL DEFAULT 0,
- PRIMARY KEY (`id`),
- UNIQUE KEY `ux_impression_id` (`impression_id`),
- INDEX `idx_experiment_id` (`experiment_id`),
- INDEX `idx_impression_id` (`impression_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-SET FOREIGN_KEY_CHECKS = 1;
-
-
--- ----------------------------
--- Table structure for `counter`
--- ----------------------------
-DROP TABLE IF EXISTS `counter`;
-CREATE TABLE `counter` (
- `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
- `use_flag` tinyint(1) NOT NULL DEFAULT '0',
- `version` smallint(1) NOT NULL DEFAULT '1',
- `service` varchar(64) NOT NULL DEFAULT '',
- `action` varchar(64) NOT NULL DEFAULT '',
- `item_type` int(11) NOT NULL DEFAULT '0',
- `auto_comb` tinyint(1) NOT NULL DEFAULT '1',
- `dimension` varchar(1024) NOT NULL,
- `use_profile` tinyint(1) NOT NULL DEFAULT '0',
- `bucket_imp_id` varchar(64) DEFAULT NULL,
- `use_exact` tinyint(1) NOT NULL DEFAULT '1',
- `use_rank` tinyint(1) NOT NULL DEFAULT '1',
- `ttl` int(11) NOT NULL DEFAULT '172800',
- `daily_ttl` int(11) DEFAULT NULL,
- `hbase_table` varchar(1024) DEFAULT NULL,
- `interval_unit` varchar(1024) DEFAULT NULL,
- `rate_action_id` int(11) unsigned DEFAULT NULL,
- `rate_base_id` int(11) unsigned DEFAULT NULL,
- `rate_threshold` int(11) DEFAULT NULL,
- PRIMARY KEY (`id`),
- UNIQUE KEY `svc` (`service`,`action`),
- KEY `rate_action_id` (`rate_action_id`),
- KEY `rate_base_id` (`rate_base_id`),
- CONSTRAINT `rate_action_id` FOREIGN KEY (`rate_action_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
- CONSTRAINT `rate_base_id` FOREIGN KEY (`rate_base_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
new file mode 100644
index 0000000..d8ee5bc
--- /dev/null
+++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
@@ -0,0 +1,227 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+SET FOREIGN_KEY_CHECKS = 0;
+
+-- ----------------------------
+-- Table structure for `services`
+-- ----------------------------
+DROP TABLE IF EXISTS `services`;
+CREATE TABLE `services` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_name` varchar(64) NOT NULL,
+ `access_token` varchar(64) NOT NULL,
+ `cluster` varchar(255) NOT NULL,
+ `hbase_table_name` varchar(255) NOT NULL,
+ `pre_split_size` integer NOT NULL default 0,
+ `hbase_table_ttl` integer,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_services_service_name` (`service_name`),
+ INDEX `idx_services_access_token` (`access_token`),
+ INDEX `idx_services_cluster` (`cluster`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+-- ----------------------------
+-- Table structure for `services_columns`
+-- ----------------------------
+DROP TABLE IF EXISTS `service_columns`;
+CREATE TABLE `service_columns` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_id` integer NOT NULL,
+ `column_name` varchar(64) NOT NULL,
+ `column_type` varchar(8) NOT NULL,
+ `schema_version` varchar(8) NOT NULL default 'v2',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_service_id_column_name` (`service_id`, `column_name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE service_columns add FOREIGN KEY(service_id) REFERENCES services(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `column_metas`
+-- ----------------------------
+DROP TABLE IF EXISTS `column_metas`;
+CREATE TABLE `column_metas` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `column_id` integer NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `seq` tinyint NOT NULL,
+ `data_type` varchar(8) NOT NULL DEFAULT 'string',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_column_id_name` (`column_id`, `name`),
+ INDEX `idx_column_id_seq` (`column_id`, `seq`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE column_metas ADD FOREIGN KEY(column_id) REFERENCES service_columns(id) ON DELETE CASCADE;
+
+-- ----------------------------
+-- Table structure for `labels`
+-- ----------------------------
+
+DROP TABLE IF EXISTS `labels`;
+CREATE TABLE `labels` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `label` varchar(64) NOT NULL,
+ `src_service_id` integer NOT NULL,
+ `src_column_name` varchar(64) NOT NULL,
+ `src_column_type` varchar(8) NOT NULL,
+ `tgt_service_id` integer NOT NULL,
+ `tgt_column_name` varchar(64) NOT NULL,
+ `tgt_column_type` varchar(8) NOT NULL,
+ `is_directed` tinyint NOT NULL DEFAULT 1,
+ `service_name` varchar(64),
+ `service_id` integer NOT NULL,
+ `consistency_level` varchar(8) NOT NULL DEFAULT 'weak',
+ `hbase_table_name` varchar(255) NOT NULL DEFAULT 's2graph',
+ `hbase_table_ttl` integer,
+ `schema_version` varchar(8) NOT NULL default 'v2',
+ `is_async` tinyint(4) NOT NULL default '0',
+ `compressionAlgorithm` varchar(64) NOT NULL DEFAULT 'lz4',
+ `options` text,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label` (`label`),
+ INDEX `idx_labels_src_column_name` (`src_column_name`),
+ INDEX `idx_labels_tgt_column_name` (`tgt_column_name`),
+ INDEX `idx_labels_src_service_id` (`src_service_id`),
+ INDEX `idx_labels_tgt_service_id` (`tgt_service_id`),
+ INDEX `idx_labels_service_name` (`service_name`),
+ INDEX `idx_labels_service_id` (`service_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id);
+
+
+
+-- ----------------------------
+-- Table structure for `label_metas`
+-- ----------------------------
+DROP TABLE IF EXISTS `label_metas`;
+CREATE TABLE `label_metas` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `label_id` integer NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `seq` tinyint NOT NULL,
+ `default_value` varchar(64) NOT NULL,
+ `data_type` varchar(8) NOT NULL DEFAULT 'long',
+ `used_in_index` tinyint NOT NULL DEFAULT 0,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label_metas_label_id_name` (`label_id`, `name`),
+ INDEX `idx_label_metas_label_id_seq` (`label_id`, `seq`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE label_metas ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `label_indices`
+-- ----------------------------
+DROP TABLE IF EXISTS `label_indices`;
+CREATE TABLE `label_indices` (
+ `id` int(11) NOT NULL AUTO_INCREMENT,
+ `label_id` int(11) NOT NULL,
+ `name` varchar(64) NOT NULL DEFAULT '_PK',
+ `seq` tinyint(4) NOT NULL,
+ `meta_seqs` varchar(64) NOT NULL,
+ `formulars` varchar(255) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_label_indices_label_id_seq` (`label_id`,`meta_seqs`),
+ UNIQUE KEY `ux_label_indices_label_id_name` (`label_id`,`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+ALTER TABLE label_indices ADD FOREIGN KEY(label_id) REFERENCES labels(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `experiments`
+-- ----------------------------
+DROP TABLE IF EXISTS `experiments`;
+CREATE TABLE `experiments` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `service_id` integer NOT NULL,
+ `service_name` varchar(128) NOT NULL,
+ `name` varchar(64) NOT NULL,
+ `description` varchar(255) NOT NULL,
+ `experiment_type` varchar(8) NOT NULL DEFAULT 'u',
+ `total_modular` int NOT NULL DEFAULT 100,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_experiments_service_id_name` (`service_id`, `name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ALTER TABLE experiments ADD FOREIGN KEY(service_id) REFERENCES service(id) ON DELETE CASCADE;
+
+
+-- ----------------------------
+-- Table structure for `buckets`
+-- ----------------------------
+DROP TABLE IF EXISTS `buckets`;
+CREATE TABLE `buckets` (
+ `id` integer NOT NULL AUTO_INCREMENT,
+ `experiment_id` integer NOT NULL,
+ `modular` varchar(64) NOT NULL,
+ `http_verb` varchar(8) NOT NULL,
+ `api_path` text NOT NULL,
+ `uuid_key` varchar(128),
+ `uuid_placeholder` varchar(64),
+ `request_body` text NOT NULL,
+ `timeout` int NOT NULL DEFAULT 1000,
+ `impression_id` varchar(64) NOT NULL,
+ `is_graph_query` tinyint NOT NULL DEFAULT 1,
+ `is_empty` tinyint NOT NULL DEFAULT 0,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `ux_buckets_impression_id` (`impression_id`),
+ INDEX `idx_buckets_experiment_id` (`experiment_id`),
+ INDEX `idx_buckets_impression_id` (`impression_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+SET FOREIGN_KEY_CHECKS = 1;
+
+
+-- ----------------------------
+-- Table structure for `counter`
+-- ----------------------------
+DROP TABLE IF EXISTS `counter`;
+CREATE TABLE `counter` (
+ `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+ `use_flag` tinyint(1) NOT NULL DEFAULT '0',
+ `version` smallint(1) NOT NULL DEFAULT '1',
+ `service` varchar(64) NOT NULL DEFAULT '',
+ `action` varchar(64) NOT NULL DEFAULT '',
+ `item_type` int(11) NOT NULL DEFAULT '0',
+ `auto_comb` tinyint(1) NOT NULL DEFAULT '1',
+ `dimension` varchar(1024) NOT NULL,
+ `use_profile` tinyint(1) NOT NULL DEFAULT '0',
+ `bucket_imp_id` varchar(64) DEFAULT NULL,
+ `use_exact` tinyint(1) NOT NULL DEFAULT '1',
+ `use_rank` tinyint(1) NOT NULL DEFAULT '1',
+ `ttl` int(11) NOT NULL DEFAULT '172800',
+ `daily_ttl` int(11) DEFAULT NULL,
+ `hbase_table` varchar(1024) DEFAULT NULL,
+ `interval_unit` varchar(1024) DEFAULT NULL,
+ `rate_action_id` int(11) unsigned DEFAULT NULL,
+ `rate_base_id` int(11) unsigned DEFAULT NULL,
+ `rate_threshold` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `counter_svc` (`service`,`action`),
+ KEY `counter_rate_action_id` (`rate_action_id`),
+ KEY `counter_rate_base_id` (`rate_base_id`),
+ CONSTRAINT `counter_rate_action_id` FOREIGN KEY (`rate_action_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION,
+ CONSTRAINT `counter_rate_base_id` FOREIGN KEY (`rate_base_id`) REFERENCES `counter` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql
new file mode 100644
index 0000000..1bda27c
--- /dev/null
+++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/setup.sql
@@ -0,0 +1,28 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+
+CREATE DATABASE IF NOT EXISTS graph_dev;
+
+CREATE USER 'graph'@'%' IDENTIFIED BY 'graph';
+
+GRANT ALL PRIVILEGES ON graph_dev.* TO 'graph'@'%' identified by 'graph';
+
+flush privileges;
+
+use graph_dev;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
index 7439939..bf6dfb7 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/Model.scala
@@ -26,8 +26,9 @@ import org.apache.s2graph.core.utils.{SafeUpdateCache, logger}
import scalikejdbc._
import scala.concurrent.ExecutionContext
+import scala.io.Source
import scala.language.{higherKinds, implicitConversions}
-import scala.util.{Failure, Try}
+import scala.util.{Success, Failure, Try}
object Model {
var maxSize = 10000
@@ -52,11 +53,42 @@ object Model {
config.getString("db.default.user"),
config.getString("db.default.password"),
settings)
+
+ checkSchema()
+ }
+
+ def checkSchema(): Unit = {
+ withTx { implicit session =>
+ sql"""show tables""".map(rs => rs.string(1)).list.apply()
+ } match {
+ case Success(tables) =>
+ if (tables.isEmpty) {
+ // this is a very simple migration tool that only supports creating
+ // appropriate tables when there are no tables in the database at all.
+ // Ideally, it should be improved to a sophisticated migration tool
+ // that supports versioning, etc.
+ logger.info("Creating tables ...")
+ val schema = getClass.getResourceAsStream("schema.sql")
+ val lines = Source.fromInputStream(schema, "UTF-8").getLines
+ val sources = lines.map(_.split("-- ").head.trim).mkString("\n")
+ val statements = sources.split(";\n")
+ withTx { implicit session =>
+ statements.foreach(sql => session.execute(sql))
+ } match {
+ case Success(_) =>
+ logger.info("Successfully imported schema")
+ case Failure(e) =>
+ throw new RuntimeException("Error while importing schema", e)
+ }
+ }
+ case Failure(e) =>
+ throw new RuntimeException("Could not list tables in the database", e)
+ }
}
def withTx[T](block: DBSession => T): Try[T] = {
using(DB(ConnectionPool.borrow())) { conn =>
- val res = Try {
+ Try {
conn.begin()
val session = conn.withinTxSession()
val result = block(session)
@@ -69,8 +101,6 @@ object Model {
conn.rollbackIfActive()
Failure(e)
}
-
- res
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2counter_core/build.sbt
----------------------------------------------------------------------
diff --git a/s2counter_core/build.sbt b/s2counter_core/build.sbt
index 4a49bd3..d4c0bdf 100644
--- a/s2counter_core/build.sbt
+++ b/s2counter_core/build.sbt
@@ -29,9 +29,13 @@ libraryDependencies ++= Seq(
"com.typesafe.play" %% "play-json" % Common.playVersion,
"com.typesafe.play" %% "play-ws" % Common.playVersion,
"com.typesafe.akka" %% "akka-actor" % "2.3.4",
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"org.apache.hbase" % "hbase-client" % Common.hbaseVersion,
"org.apache.hbase" % "hbase-common" % Common.hbaseVersion,
+ "org.apache.hbase" % "hbase-hadoop-compat" % Common.hbaseVersion,
+ "org.apache.hbase" % "hbase-hadoop2-compat" % Common.hbaseVersion,
"org.apache.hadoop" % "hadoop-common" % Common.hadoopVersion,
+ "org.apache.hadoop" % "hadoop-hdfs" % Common.hadoopVersion,
"redis.clients" % "jedis" % "2.6.0",
"org.apache.kafka" % "kafka-clients" % "0.8.2.0",
"mysql" % "mysql-connector-java" % "5.1.28",
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2counter_loader/build.sbt
----------------------------------------------------------------------
diff --git a/s2counter_loader/build.sbt b/s2counter_loader/build.sbt
index 879f2c1..27f08f5 100644
--- a/s2counter_loader/build.sbt
+++ b/s2counter_loader/build.sbt
@@ -26,6 +26,7 @@ version := "0.12.1-SNAPSHOT"
scalacOptions in Test ++= Seq("-Yrangepos")
libraryDependencies ++= Seq(
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion,
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_netty/build.sbt b/s2rest_netty/build.sbt
index c810482..b7c5484 100644
--- a/s2rest_netty/build.sbt
+++ b/s2rest_netty/build.sbt
@@ -22,5 +22,6 @@ name := "s2rest_netty"
enablePlugins(JavaAppPackaging)
libraryDependencies ++= Seq(
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"io.netty" % "netty-all" % "4.0.33.Final"
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/conf/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/conf/reference.conf b/s2rest_netty/conf/reference.conf
index a50e7b2..9b37eda 100644
--- a/s2rest_netty/conf/reference.conf
+++ b/s2rest_netty/conf/reference.conf
@@ -131,7 +131,7 @@ hbase.fail.prob=-1.0
# max allowd edges for deleteAll is multiply of above two configuration.
# set global obejct package, TODO: remove global
-application.global=com.kakao.s2graph.rest.Global
+#application.global=com.kakao.s2graph.rest.Global
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_netty/src/main/resources/reference.conf
----------------------------------------------------------------------
diff --git a/s2rest_netty/src/main/resources/reference.conf b/s2rest_netty/src/main/resources/reference.conf
index a50e7b2..9b37eda 100644
--- a/s2rest_netty/src/main/resources/reference.conf
+++ b/s2rest_netty/src/main/resources/reference.conf
@@ -131,7 +131,7 @@ hbase.fail.prob=-1.0
# max allowd edges for deleteAll is multiply of above two configuration.
# set global obejct package, TODO: remove global
-application.global=com.kakao.s2graph.rest.Global
+#application.global=com.kakao.s2graph.rest.Global
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/s2rest_play/build.sbt
----------------------------------------------------------------------
diff --git a/s2rest_play/build.sbt b/s2rest_play/build.sbt
index 27922f0..6edb8e8 100644
--- a/s2rest_play/build.sbt
+++ b/s2rest_play/build.sbt
@@ -24,6 +24,7 @@ scalacOptions in Test ++= Seq("-Yrangepos")
libraryDependencies ++= Seq(
ws,
filters,
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"com.github.danielwegener" % "logback-kafka-appender" % "0.0.3"
)
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/fabb212d/spark/build.sbt
----------------------------------------------------------------------
diff --git a/spark/build.sbt b/spark/build.sbt
index 2c2a389..f5412d8 100644
--- a/spark/build.sbt
+++ b/spark/build.sbt
@@ -22,6 +22,7 @@ name := "s2spark"
scalacOptions ++= Seq("-deprecation", "-feature")
libraryDependencies ++= Seq(
+ "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
"org.apache.spark" %% "spark-core" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming" % Common.sparkVersion % "provided",
"org.apache.spark" %% "spark-streaming-kafka" % Common.sparkVersion,