You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2018/03/14 10:16:56 UTC

[1/5] incubator-s2graph git commit: - add s2jobs subproject. - migrate bulk loader with spark 2.3.0. - add test cases for bulk loader.

Repository: incubator-s2graph
Updated Branches:
  refs/heads/master 042410d5b -> ddfd10daa


http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/loader/GraphFileGeneratorTest.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/loader/GraphFileGeneratorTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/loader/GraphFileGeneratorTest.scala
new file mode 100644
index 0000000..32c5a25
--- /dev/null
+++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/loader/GraphFileGeneratorTest.scala
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+import java.io.{File, PrintWriter}
+import java.util
+
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles
+import org.apache.hadoop.util.ToolRunner
+import org.apache.s2graph.core.{Management, PostProcess, S2Graph, S2VertexLike}
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.mysqls.{Label, ServiceColumn}
+import org.apache.s2graph.core.storage.CanSKeyValue
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.s2graph.s2jobs.S2GraphHelper
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers}
+import play.api.libs.json.Json
+
+import scala.io.Source
+import scala.util.Try
+
+object GraphFileGeneratorTest {
+  def initTestEdgeSchema(s2: S2Graph, tableName: String,
+                         schemaVersion: String = HBaseType.DEFAULT_VERSION,
+                         compressionAlgorithm: String = "none"): Label = {
+    import scala.collection.JavaConverters._
+    /* initialize model for test */
+    val management = s2.management
+
+    val service = management.createService(serviceName = "s2graph", cluster = "localhost",
+      hTableName = "s2graph", preSplitSize = -1, hTableTTL = -1, compressionAlgorithm = "gz")
+
+    val serviceColumn = management.createServiceColumn(service.serviceName, "user", "string", Nil)
+
+    Try {
+      management.createLabel("friends", serviceColumn, serviceColumn, isDirected = true,
+        serviceName = service.serviceName, indices = new java.util.ArrayList[Index],
+        props = Seq(Prop("since", "0", "long"), Prop("score", "0", "integer")).asJava, consistencyLevel = "strong", hTableName = tableName,
+        hTableTTL = -1, schemaVersion = schemaVersion, compressionAlgorithm = compressionAlgorithm, options = "")
+    }
+
+    Label.findByName("friends").getOrElse(throw new IllegalArgumentException("friends label is not initialized."))
+  }
+
+  def initTestVertexSchema(s2: S2Graph): ServiceColumn = {
+    import scala.collection.JavaConverters._
+    /* initialize model for test */
+    val management = s2.management
+
+    val service = management.createService(serviceName = "device_profile", cluster = "localhost",
+      hTableName = "s2graph", preSplitSize = -1, hTableTTL = -1, compressionAlgorithm = "gz")
+
+    management.createServiceColumn(service.serviceName, "imei", "string",
+      Seq(
+        Prop(name = "first_time", defaultValue = "''", dataType = "string"),
+        Prop(name = "last_time", defaultValue = "''", dataType = "string"),
+        Prop(name = "total_active_days", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "query_amount", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "active_months", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "fua", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_province", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_city", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_days", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "location_last_province", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_last_city", defaultValue = "''", dataType = "string"),
+        Prop(name = "fimei_legality", defaultValue = "-1", dataType = "integer")
+      ))
+  }
+
+  def writeToFile(fileName: String)(lines: Seq[String]): Unit = {
+    val writer = new PrintWriter(fileName)
+    lines.foreach(line => writer.write(line + "\n"))
+    writer.close
+  }
+
+  def deleteRecursively(file: File): Unit = {
+    if (file.isDirectory) file.listFiles.foreach(deleteRecursively)
+    if (file.exists && !file.delete) throw new Exception(s"Unable to delete ${file.getAbsolutePath}")
+  }
+}
+
+class GraphFileGeneratorTest extends FunSuite with Matchers with BeforeAndAfterAll {
+  import GraphFileGeneratorTest._
+  import scala.collection.JavaConverters._
+
+  private val master = "local[2]"
+  private val appName = "example-spark"
+
+  private var sc: SparkContext = _
+  val options = GraphFileOptions(
+    input = "/tmp/imei-20.txt",
+    tempDir = "/tmp/bulkload_tmp",
+    output = "/tmp/s2graph_bulkload",
+    zkQuorum = "localhost",
+    dbUrl = "jdbc:h2:file:./var/metastore;MODE=MYSQL",
+    dbUser = "sa",
+    dbPassword = "sa",
+    dbDriver = "org.h2.Driver",
+    tableName = "s2graph",
+    maxHFilePerRegionServer = 1,
+    numRegions = 3,
+    compressionAlgorithm = "NONE",
+    buildDegree = false,
+    autoEdgeCreate = false)
+
+  val s2Config = Management.toConfig(options.toConfigParams)
+
+  val tableName = options.tableName
+  val schemaVersion = HBaseType.DEFAULT_VERSION
+  val compressionAlgorithm: String = options.compressionAlgorithm
+  var s2: S2Graph = _
+
+  override def beforeAll(): Unit = {
+    // initialize spark context.
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName(appName)
+
+    sc = new SparkContext(conf)
+
+    s2 = S2GraphHelper.initS2Graph(s2Config)
+  }
+
+  override def afterAll(): Unit = {
+    if (sc != null) sc.stop()
+    if (s2 != null) s2.shutdown()
+  }
+
+  test("test generateKeyValues edge only.") {
+    import scala.collection.JavaConverters._
+    import org.apache.s2graph.core.storage.CanSKeyValue._
+
+    val label = initTestEdgeSchema(s2, tableName, schemaVersion, compressionAlgorithm)
+    /* end of initialize model */
+
+    val bulkEdgeString = "1416236400000\tinsert\tedge\ta\tb\tfriends\t{\"since\":1316236400000,\"score\":10}"
+    val input = sc.parallelize(Seq(bulkEdgeString))
+
+    val kvs = HFileGenerator.transfer(sc, s2Config, input, options)
+
+    val ls = kvs.map(kv => CanSKeyValue.hbaseKeyValue.toSKeyValue(kv)).collect().toList
+
+    val serDe = s2.defaultStorage.serDe
+
+    val bulkEdge = s2.elementBuilder.toGraphElement(bulkEdgeString, options.labelMapping).get
+
+    val indexEdges = ls.flatMap { kv =>
+      serDe.indexEdgeDeserializer(label.schemaVersion).fromKeyValues(Seq(kv), None)
+    }
+
+    val indexEdge = indexEdges.head
+
+    println(indexEdge)
+    println(bulkEdge)
+
+    bulkEdge shouldBe(indexEdge)
+  }
+
+
+  test("test generateKeyValues vertex only.") {
+    val serviceColumn = initTestVertexSchema(s2)
+    val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\tdevice_profile\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广东省\",\"location_often_city\":\"深圳市\",\"location_often_days\":6,\"location_last_province\":\"广东省\",\"location_last_city\":\"深圳市\",\"fimei_legality\":3}"
+    val bulkVertex = s2.elementBuilder.toGraphElement(bulkVertexString, options.labelMapping).get
+
+    val input = sc.parallelize(Seq(bulkVertexString))
+
+    val kvs = HFileGenerator.transfer(sc, s2Config, input, options)
+
+    val ls = kvs.map(kv => CanSKeyValue.hbaseKeyValue.toSKeyValue(kv)).collect().toList
+
+    val serDe = s2.defaultStorage.serDe
+
+    val vertex = serDe.vertexDeserializer(serviceColumn.schemaVersion).fromKeyValues(ls, None).get
+
+    PostProcess.s2VertexToJson(vertex).foreach { jsValue =>
+      println(Json.prettyPrint(jsValue))
+    }
+
+    bulkVertex shouldBe(vertex)
+  }
+
+  test("test generateHFile vertex only.") {
+    val serviceColumn = initTestVertexSchema(s2)
+//    val lines = Source.fromFile("/tmp/imei-20.txt").getLines().toSeq
+    val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\ts2graph\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广东省\",\"location_often_city\":\"深圳市\",\"location_often_days\":6,\"location_last_province\":\"广东省\",\"location_last_city\":\"深圳市\",\"fimei_legality\":3}"
+    val lines = Seq(bulkVertexString)
+    val input = sc.parallelize(lines)
+
+    val kvs = HFileGenerator.transfer(sc, s2Config, input, options)
+    println(kvs.count())
+  }
+
+  // this test case expect options.input already exist with valid bulk load format.
+  test("bulk load and fetch vertex: spark mode") {
+    val serviceColumn = initTestVertexSchema(s2)
+
+    deleteRecursively(new File(options.tempDir))
+    deleteRecursively(new File(options.output))
+
+    val bulkVertexLs = Source.fromFile(options.input).getLines().toSeq
+    val input = sc.parallelize(bulkVertexLs)
+
+    HFileGenerator.generate(sc, s2Config, input, options)
+
+    val hfileArgs = Array(options.output, options.tableName)
+    val hbaseConfig = HBaseConfiguration.create()
+
+    val ret = ToolRunner.run(hbaseConfig, new LoadIncrementalHFiles(hbaseConfig), hfileArgs)
+
+    val s2Vertices = s2.vertices().asScala.toSeq.map(_.asInstanceOf[S2VertexLike])
+    val json = PostProcess.verticesToJson(s2Vertices)
+
+    println(Json.prettyPrint(json))
+  }
+
+  // this test case expect options.input already exist with valid bulk load format.
+  test("bulk load and fetch vertex: mr mode") {
+    val serviceColumn = initTestVertexSchema(s2)
+
+    deleteRecursively(new File(options.tempDir))
+    deleteRecursively(new File(options.output))
+
+    val bulkVertexLs = Source.fromFile(options.input).getLines().toSeq
+    val input = sc.parallelize(bulkVertexLs)
+
+    HFileMRGenerator.generate(sc, s2Config, input, options)
+
+    val hfileArgs = Array(options.output, options.tableName)
+    val hbaseConfig = HBaseConfiguration.create()
+
+    val ret = ToolRunner.run(hbaseConfig, new LoadIncrementalHFiles(hbaseConfig), hfileArgs)
+    val s2Vertices = s2.vertices().asScala.toSeq.map(_.asInstanceOf[S2VertexLike])
+    val json = PostProcess.verticesToJson(s2Vertices)
+
+    println(Json.prettyPrint(json))
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2rest_play/conf/application.conf
----------------------------------------------------------------------
diff --git a/s2rest_play/conf/application.conf b/s2rest_play/conf/application.conf
index fe95886..5fb2552 100644
--- a/s2rest_play/conf/application.conf
+++ b/s2rest_play/conf/application.conf
@@ -15,4 +15,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-#
+#
\ No newline at end of file


[5/5] incubator-s2graph git commit: [S2GRAPH-182]: Version up spark dependencies.

Posted by st...@apache.org.
[S2GRAPH-182]: Version up spark dependencies.

JIRA:
    [S2GRAPH-182] https://issues.apache.org/jira/browse/S2GRAPH-182

Pull Request:
    Closes #136

Author
    DO YUNG YOON <st...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/ddfd10da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/ddfd10da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/ddfd10da

Branch: refs/heads/master
Commit: ddfd10daa42d8c5f291104c7a8db5298760b9758
Parents: 041a137 042410d
Author: DO YUNG YOON <st...@apache.org>
Authored: Wed Mar 14 18:14:29 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Wed Mar 14 19:16:24 2018 +0900

----------------------------------------------------------------------
 CHANGES | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/ddfd10da/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index 1f72a38,1f72a38..c44c8c0
--- a/CHANGES
+++ b/CHANGES
@@@ -52,6 -52,6 +52,7 @@@ Release Notes - S2Graph - Version 0.2.
      * [S2GRAPH-167] - Provide Vertex CSV File BulkLoad
      * [S2GRAPH-169] - Separate multiple functionalities on Storage class into multiple Interface.
      * [S2GRAPH-170] - Create Interface for S2Edge/S2Vertex/S2Graph.
++    * [S2GRAPH-182] - Version up spark dependencies.
  
  ** New Feature
      * [S2GRAPH-123] - Support different index on out/in direction.


[3/5] incubator-s2graph git commit: - add s2jobs subproject. - migrate bulk loader with spark 2.3.0. - add test cases for bulk loader.

Posted by st...@apache.org.
- add s2jobs subproject.
- migrate bulk loader with spark 2.3.0.
- add test cases for bulk loader.


Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/db7f0191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/db7f0191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/db7f0191

Branch: refs/heads/master
Commit: db7f01914073b8f56f64c243d797ee538c1ffc51
Parents: aeaff3f
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Mar 6 15:01:50 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Mar 6 16:02:41 2018 +0900

----------------------------------------------------------------------
 CHANGES                                         |   1 +
 build.sbt                                       |  11 +-
 loader/build.sbt                                |   5 +-
 loader/loader.py                                |  85 +-
 .../s2graph/loader/spark/HBaseContext.scala     |  12 +-
 .../loader/spark/HBaseRDDFunctions.scala        |   3 +-
 .../loader/subscriber/GraphSubscriber.scala     |  10 +
 .../loader/subscriber/TransferToHFile.scala     | 278 ++++--
 .../loader/subscriber/TransferToHFileTest.scala | 232 +++++
 project/Common.scala                            |   1 +
 .../s2graph/core/GraphElementBuilder.scala      |   2 +-
 .../org/apache/s2graph/core/JSONParser.scala    |   3 +-
 .../s2graph/core/mysqls/ServiceColumn.scala     |   4 +-
 .../apache/s2graph/core/storage/SKeyValue.scala |   4 +
 .../s2graph/core/storage/StorageSerDe.scala     |   2 +-
 .../hbase/AsynchbaseStorageManagement.scala     |  16 +-
 s2jobs/build.sbt                                |  57 ++
 s2jobs/loader.py                                | 153 ++++
 .../hbase/mapreduce/GraphHFileOutputFormat.java | 169 ++++
 .../apache/s2graph/s2jobs/S2GraphHelper.scala   |  31 +
 .../s2jobs/loader/GraphFileGenerator.scala      |  42 +
 .../s2jobs/loader/GraphFileOptions.scala        | 138 +++
 .../s2graph/s2jobs/loader/HFileGenerator.scala  | 222 +++++
 .../s2jobs/loader/HFileMRGenerator.scala        | 161 ++++
 .../s2jobs/loader/RawFileGenerator.scala        |  51 ++
 .../s2jobs/spark/BulkLoadPartitioner.scala      |  56 ++
 .../s2jobs/spark/FamilyHFileWriteOptions.scala  |  35 +
 .../s2graph/s2jobs/spark/HBaseContext.scala     | 850 +++++++++++++++++++
 .../s2jobs/spark/HBaseDStreamFunctions.scala    | 158 ++++
 .../s2jobs/spark/HBaseRDDFunctions.scala        | 206 +++++
 .../s2graph/s2jobs/spark/JavaHBaseContext.scala | 342 ++++++++
 .../s2jobs/spark/KeyFamilyQualifier.scala       |  46 +
 .../s2graph/s2jobs/S2GraphHelperTest.scala      |  24 +
 .../s2jobs/loader/GraphFileGeneratorTest.scala  | 256 ++++++
 s2rest_play/conf/application.conf               |   2 +-
 35 files changed, 3546 insertions(+), 122 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index b54aaeb..1f72a38 100644
--- a/CHANGES
+++ b/CHANGES
@@ -39,6 +39,7 @@ Release Notes - S2Graph - Version 0.2.0
     * [S2GRAPH-168] - Fix args order mismatch when use addServiceColumnProp
     * [S2GRAPH-176] - Fix compile error on LabelMeta
     * [S2GRAPH-179] - Add defaultValue on ColumnMeta
+    * [S2GRAPH-178] - Fix null pointer error on BulkLoader
 
 ** Improvement
     * [S2GRAPH-72] - Support Apache TinkerPop and Gremlin

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/build.sbt
----------------------------------------------------------------------
diff --git a/build.sbt b/build.sbt
index cdabfbf..af6fbd7 100755
--- a/build.sbt
+++ b/build.sbt
@@ -56,8 +56,8 @@ lazy val s2core = project.settings(commonSettings: _*)
 
 lazy val spark = project.settings(commonSettings: _*)
 
-lazy val loader = project.dependsOn(s2core, spark)
-  .settings(commonSettings: _*)
+//lazy val loader = project.dependsOn(s2core, spark)
+//  .settings(commonSettings: _*)
 
 lazy val s2counter_core = project.dependsOn(s2core)
   .settings(commonSettings: _*)
@@ -65,12 +65,15 @@ lazy val s2counter_core = project.dependsOn(s2core)
 lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
   .settings(commonSettings: _*)
 
+lazy val s2jobs = project.dependsOn(s2core)
+  .settings(commonSettings: _*)
+
 lazy val s2graph_gremlin = project.dependsOn(s2core)
   .settings(commonSettings: _*)
 
 lazy val root = (project in file("."))
   .aggregate(s2core, s2rest_play)
-  .dependsOn(s2rest_play, s2rest_netty, loader, s2counter_loader, s2graphql) // this enables packaging on the root project
+  .dependsOn(s2rest_play, s2rest_netty, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project
   .settings(commonSettings: _*)
 
 lazy val runRatTask = inputKey[Unit]("Runs Apache rat on S2Graph")
@@ -88,7 +91,7 @@ publishSigned := {
   (publishSigned in s2rest_netty).value
   (publishSigned in s2core).value
   (publishSigned in spark).value
-  (publishSigned in loader).value
+  (publishSigned in s2jobs).value
   (publishSigned in s2counter_core).value
   (publishSigned in s2counter_loader).value
 }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/build.sbt
----------------------------------------------------------------------
diff --git a/loader/build.sbt b/loader/build.sbt
index ac7d948..a93713a 100644
--- a/loader/build.sbt
+++ b/loader/build.sbt
@@ -29,14 +29,15 @@ projectDependencies := Seq(
 
 libraryDependencies ++= Seq(
   "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
-  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
+  "org.apache.spark" %% "spark-core" % sparkVersion % "provided" exclude("javax.servlet", "*"),
   "org.apache.spark" %% "spark-streaming" % sparkVersion % "provided",
   "org.apache.spark" %% "spark-hive" % sparkVersion % "provided",
   "org.apache.spark" %% "spark-streaming-kafka" % sparkVersion,
   "org.apache.httpcomponents" % "fluent-hc" % "4.2.5",
   "org.specs2" %% "specs2-core" % specs2Version % "test",
   "org.scalatest" %% "scalatest" % "2.2.1" % "test",
-  "org.apache.hadoop" % "hadoop-distcp" % hadoopVersion
+  "org.apache.hadoop" % "hadoop-distcp" % hadoopVersion,
+  "com.github.scopt" %% "scopt" % "3.7.0"
 )
 
 crossScalaVersions := Seq("2.10.6")

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/loader.py
----------------------------------------------------------------------
diff --git a/loader/loader.py b/loader/loader.py
index 1d4dc32..634f2be 100644
--- a/loader/loader.py
+++ b/loader/loader.py
@@ -16,50 +16,76 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os, sys, urllib2, urllib
+import os, sys
+#, urllib2, urllib
 
 def cleanup(args): 
 	cmd = "hadoop fs -rm -r /tmp/%s" % args["htable_name"]
-	print cmd
+	print(cmd)
 	ret = os.system(cmd)
-	print cmd, "return", ret
+	print(cmd, "return", ret)
 	return ret
 
 def hfile(args):
-	cmd = """spark-submit --class "subscriber.TransferToHFile" \
+	cmd = """spark-submit --class "org.apache.s2graph.loader.subscriber.TransferToHFile" \
 --name "TransferToHFile@shon" \
 --conf "spark.task.maxFailures=20" \
 --master yarn-cluster \
---num-executors %s --driver-memory 1g --executor-memory 2g --executor-cores 1 %s \
-%s /tmp/%s %s %s %s %s %s %s""" % (args["num_executors"], JAR, args["input"], args["htable_name"], args["hbase_zk"], args["htable_name"], args["db_url"], args["max_file_per_region"], args["label_mapping"], args["auto_create_edge"])
-	print cmd
+--num-executors %s \
+--driver-memory 1g \
+--executor-memory 2g \
+--executor-cores 1 \
+%s \
+--input %s \
+--tmpPath /tmp/%s \
+--zkQuorum %s \
+--table %s \
+--dbUrl %s \
+--dbUser %s \
+--dbPassword %s \
+--maxHFilePerRegionServer %s \
+--labelMapping %s \
+--autoEdgeCreate %s""" % (args["num_executors"],
+						   JAR,
+						   args["input"],
+						   args["htable_name"],
+						   args["hbase_zk"],
+						   args["htable_name"],
+						   args["db_url"],
+						   args["db_user"],
+						   args["db_password"],
+						   args["max_file_per_region"],
+						   args["label_mapping"],
+						   args["auto_create_edge"])
+	print(cmd)
 	ret = os.system(cmd)
-	print cmd, "return", ret
+	print(cmd, "return", ret)
 	return ret
 
 def distcp(args): 
 	cmd = "hadoop distcp -overwrite -m %s -bandwidth %s /tmp/%s %s/tmp/%s" % (args["-m"], args["-bandwidth"], args["htable_name"], args["hbase_namenode"], args["htable_name"])
-	print cmd
+	print(cmd)
 	ret = os.system(cmd)
-	print cmd, "return", ret
+	print(cmd, "return", ret)
 	return ret
 
 def chmod(args):
 	cmd = "export HADOOP_CONF_DIR=%s; export HADOOP_USER_NAME=hdfs; hadoop fs -chmod -R 777 /tmp/%s" % (args["HADOOP_CONF_DIR"], args["htable_name"])
-	print cmd
+	print(cmd)
 	ret = os.system(cmd)
-	print cmd, "return", ret
+	print(cmd, "return", ret)
 	return ret
 
 def load(args):
-	cmd = "export HADOOP_CONF_DIR=%s; export HBASE_CONF_DIR=%s; hbase %s /tmp/%s %s" % (args["HADOOP_CONF_DIR"], args["HBASE_CONF_DIR"], LOADER_CLASS, args["htable_name"], args["htable_name"])
-	print cmd
+	cmd = "export HADOOP_CONF_DIR=%s; export HBASE_CONF_DIR=%s; hbase %s /tmp/%s %s" % \
+		  (args["HADOOP_CONF_DIR"], args["HBASE_CONF_DIR"], LOADER_CLASS, args["htable_name"], args["htable_name"])
+	print(cmd)
 	ret = os.system(cmd)
-	print cmd, "return", ret
+	print(cmd, "return", ret)
 	return ret
 
 def send(msg):
-	print msg
+	print(msg)
 
 def run(args):
 	cleanup(args)
@@ -69,15 +95,15 @@ def run(args):
 	if ret != 0: return send("[Failed]: loader build hfile failed %s" % ret)
 	else: send("[Success]: loader build hfile")
 	
-	ret = distcp(args)
-
-	if ret != 0: return send("[Failed]: loader distcp failed %s" % ret)
-	else: send("[Success]: loader distcp")
-
-	ret = chmod(args)
-	
-	if ret != 0: return send("[Failed]: loader chmod failed %s" % ret)
-	else: send("[Success]: loader chmod")
+	# ret = distcp(args)
+    #
+	# if ret != 0: return send("[Failed]: loader distcp failed %s" % ret)
+	# else: send("[Success]: loader distcp")
+    #
+	# ret = chmod(args)
+	#
+	# if ret != 0: return send("[Failed]: loader chmod failed %s" % ret)
+	# else: send("[Success]: loader chmod")
 
 	ret = load(args)
 
@@ -86,23 +112,24 @@ def run(args):
 
 
 LOADER_CLASS = "org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles"
-JAR="loader/target/scala-2.10/s2loader-assembly-0.11.0-SNAPSHOT.jar"
-
+JAR="loader/target/scala-2.11/s2loader-assembly-0.2.1-SNAPSHOT.jar"
 
 args = {
 "HADOOP_CONF_DIR": "hdfs_conf_gasan", 
 "HBASE_CONF_DIR": "hbase_conf_gasan", 
 "htable_name": "test", 
-"hbase_namenode": "hdfs://nameservice:8020",
+"hbase_namenode": "hdfs://localhost:8020",
 "hbase_zk": "localhost",
 "db_url": "jdbc:mysql://localhost:3306/graph_dev",
+"db_user": "sa",
+"db_password": "sa",
 "max_file_per_region": 1,
 "label_mapping": "none",
 "auto_create_edge": "false",
 "-m": 1, 
 "-bandwidth": 10,
 "num_executors": 2,
-"input": "/user/test.txt"
+"input": "/tmp/test.txt"
 }
 
 run(args)

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
index 1f68dc2..e177196 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseContext.scala
@@ -591,6 +591,7 @@ class HBaseContext(@transient private val sc: SparkContext,
    *
    * @param rdd                            The RDD we are bulk loading from
    * @param tableName                      The HBase table we are loading into
+   * @param startKeys
    * @param flatMap                        A flapMap function that will make every
    *                                       row in the RDD
    *                                       into N cells for the bulk load
@@ -603,17 +604,16 @@ class HBaseContext(@transient private val sc: SparkContext,
    */
   def bulkLoad[T](rdd:RDD[T],
                   tableName: TableName,
+                  startKeys: Array[Array[Byte]],
                   flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
                   stagingDir:String,
-                  familyHFileWriteOptionsMap:
-                  util.Map[Array[Byte], FamilyHFileWriteOptions] =
-                  new util.HashMap[Array[Byte], FamilyHFileWriteOptions],
+                  familyHFileWriteOptionsMap: util.Map[Array[Byte], FamilyHFileWriteOptions] = new util.HashMap[Array[Byte], FamilyHFileWriteOptions],
                   compactionExclude: Boolean = false,
                   maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):
   Unit = {
-    val conn = ConnectionFactory.createConnection(config)
-    val regionLocator = conn.getRegionLocator(tableName)
-    val startKeys = regionLocator.getStartKeys
+//    val conn = ConnectionFactory.createConnection(config)
+//    val regionLocator = conn.getRegionLocator(tableName)
+//    val startKeys = regionLocator.getStartKeys
     val defaultCompressionStr = config.get("hfile.compression",
       Compression.Algorithm.NONE.getName)
     val defaultCompression = Compression.getCompressionAlgorithmByName(defaultCompressionStr)

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
index b818a3c..54949a8 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/spark/HBaseRDDFunctions.scala
@@ -192,6 +192,7 @@ object HBaseRDDFunctions
      */
     def hbaseBulkLoad(hc: HBaseContext,
                          tableName: TableName,
+                         startKeys: Array[Array[Byte]],
                          flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
                          stagingDir:String,
                          familyHFileWriteOptionsMap:
@@ -199,7 +200,7 @@ object HBaseRDDFunctions
                          new util.HashMap[Array[Byte], FamilyHFileWriteOptions](),
                          compactionExclude: Boolean = false,
                          maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):Unit = {
-      hc.bulkLoad(rdd, tableName,
+      hc.bulkLoad(rdd, tableName, startKeys,
         flatMap, stagingDir, familyHFileWriteOptionsMap,
         compactionExclude, maxSize)
     }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
index 6ecb070..90e8bbb 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/GraphSubscriber.scala
@@ -76,6 +76,16 @@ object GraphSubscriberHelper extends WithKafka {
     }
   }
 
+  def apply(_config: Config): Unit = {
+    config = _config
+    if (g == null) {
+      val ec = ExecutionContext.Implicits.global
+      g = new S2Graph(config)(ec)
+      management = new Management(g)
+      builder = g.elementBuilder
+    }
+  }
+
   def apply(phase: String, dbUrl: String, zkQuorum: String, kafkaBrokerList: String): Unit = {
     config = GraphConfig(phase, toOption(dbUrl), toOption(zkQuorum), toOption(kafkaBrokerList))
 

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
----------------------------------------------------------------------
diff --git a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
index 6aaf6fd..bfb5a96 100644
--- a/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
+++ b/loader/src/main/scala/org/apache/s2graph/loader/subscriber/TransferToHFile.scala
@@ -19,44 +19,115 @@
 
 package org.apache.s2graph.loader.subscriber
 
-import org.apache.hadoop.hbase.client.Put
+import com.typesafe.config.Config
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase._
+import org.apache.hadoop.hbase.client.ConnectionFactory
 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
-import org.apache.hadoop.hbase.mapreduce.{TableOutputFormat}
-import org.apache.hadoop.hbase._
+import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
 import org.apache.hadoop.hbase.regionserver.BloomType
 import org.apache.hadoop.hbase.util.Bytes
 import org.apache.s2graph.core._
 import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
-import org.apache.s2graph.core.types.{InnerValLikeWithTs, SourceVertexId, LabelWithDirection}
-import org.apache.s2graph.loader.spark.{KeyFamilyQualifier, HBaseContext, FamilyHFileWriteOptions}
+import org.apache.s2graph.core.storage.hbase.AsynchbaseStorageManagement
+import org.apache.s2graph.core.types.{InnerValLikeWithTs, SourceVertexId}
+import org.apache.s2graph.loader.spark.{FamilyHFileWriteOptions, HBaseContext, KeyFamilyQualifier}
 import org.apache.s2graph.spark.spark.SparkApp
-import org.apache.spark.{SparkContext}
+import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
-import org.hbase.async.{PutRequest}
+import org.hbase.async.PutRequest
 import play.api.libs.json.Json
+
 import scala.collection.JavaConversions._
 
 
 object TransferToHFile extends SparkApp {
 
-  val usages =
-    s"""
-       |create HFiles for hbase table on zkQuorum specified.
-       |note that hbase table is created already and pre-splitted properly.
-       |
-       |params:
-       |0. input: hdfs path for tsv file(bulk format).
-       |1. output: hdfs path for storing HFiles.
-       |2. zkQuorum: running hbase cluster zkQuorum.
-       |3. tableName: table name for this bulk upload.
-       |4. dbUrl: db url for parsing to graph element.
-     """.stripMargin
+  var options:GraphFileOptions = _
+
+  case class GraphFileOptions(input: String = "",
+                              tmpPath: String = s"/tmp/bulkload",
+                              zkQuorum: String = "",
+                              tableName: String = "",
+                              dbUrl: String = "",
+                              dbUser: String = "",
+                              dbPassword: String = "",
+                              maxHFilePerRegionServer: Int = 1,
+                              numRegions: Int = 3,
+                              labelMapping: Map[String, String] = Map.empty[String, String],
+                              autoEdgeCreate: Boolean = false,
+                              buildDegree: Boolean = false,
+                              incrementalLoad: Boolean = false,
+                              compressionAlgorithm: String = "NONE") {
+    def toConfigParams = {
+      Map(
+        "hbase.zookeeper.quorum" -> zkQuorum,
+        "db.default.url" -> dbUrl,
+        "db.default.user" -> dbUser,
+        "db.default.password" -> dbPassword
+      )
+    }
+  }
+
+  val parser = new scopt.OptionParser[GraphFileOptions]("run") {
+
+    opt[String]('i', "input").required().action( (x, c) =>
+      c.copy(input = x) ).text("hdfs path for tsv file(bulk format)")
+
+    opt[String]('m', "tmpPath").required().action( (x, c) =>
+      c.copy(tmpPath = x) ).text("temp hdfs path for storing HFiles")
+
+    opt[String]('z', "zkQuorum").required().action( (x, c) =>
+      c.copy(zkQuorum = x) ).text("zookeeper config for hbase")
+
+    opt[String]('t', "table").required().action( (x, c) =>
+      c.copy(tableName = x) ).text("table name for this bulk upload.")
+
+    opt[String]('c', "dbUrl").required().action( (x, c) =>
+      c.copy(dbUrl = x)).text("jdbc connection url.")
+
+    opt[String]('u', "dbUser").required().action( (x, c) =>
+      c.copy(dbUser = x)).text("database user name.")
+
+    opt[String]('p', "dbPassword").required().action( (x, c) =>
+      c.copy(dbPassword = x)).text("database password.")
+
+    opt[Int]('h', "maxHFilePerRegionServer").action ( (x, c) =>
+      c.copy(maxHFilePerRegionServer = x)).text("maximum number of HFile per RegionServer."
+    )
+
+    opt[Int]('n', "numRegions").action ( (x, c) =>
+      c.copy(numRegions = x)).text("total numRegions(pre-split size) on table."
+    )
+
+    opt[String]('l', "labelMapping").action( (x, c) =>
+      c.copy(labelMapping = toLabelMapping(x)) ).text("mapping info to change the label from source (originalLabel:newLabel)")
+
+    opt[Boolean]('d', "buildDegree").action( (x, c) =>
+      c.copy(buildDegree = x)).text("generate degree values")
+
+    opt[Boolean]('a', "autoEdgeCreate").action( (x, c) =>
+      c.copy(autoEdgeCreate = x)).text("generate reverse edge automatically")
+
+    opt[Boolean]('c', "incrementalLoad").action( (x, c) =>
+      c.copy(incrementalLoad = x)).text("whether incremental bulkload which append data on existing table or not."
+    )
+  }
 
   //TODO: Process AtomicIncrementRequest too.
   /** build key values */
   case class DegreeKey(vertexIdStr: String, labelName: String, direction: String)
 
+  private def toLabelMapping(lableMapping: String): Map[String, String] = {
+    (for {
+      token <- lableMapping.split(",")
+      inner = token.split(":") if inner.length == 2
+    } yield {
+      (inner.head, inner.last)
+    }).toMap
+  }
+
   private def insertBulkForLoaderAsync(edge: S2Edge, createRelEdges: Boolean = true): List[PutRequest] = {
     val relEdges = if (createRelEdges) edge.relatedEdges else List(edge)
     buildPutRequests(edge.toSnapshotEdge) ++ relEdges.toList.flatMap { e =>
@@ -83,14 +154,17 @@ object TransferToHFile extends SparkApp {
       output <- List(degreeKey -> 1L) ++ extra
     } yield output
   }
+
   def buildPutRequests(snapshotEdge: SnapshotEdge): List[PutRequest] = {
     val kvs = GraphSubscriberHelper.g.getStorage(snapshotEdge.label).serDe.snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
     kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
   }
+
   def buildPutRequests(indexEdge: IndexEdge): List[PutRequest] = {
     val kvs = GraphSubscriberHelper.g.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.toList
     kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
   }
+
   def buildDegreePutRequests(vertexId: String, labelName: String, direction: String, degreeVal: Long): List[PutRequest] = {
     val label = Label.findByName(labelName).getOrElse(throw new RuntimeException(s"$labelName is not found in DB."))
     val dir = GraphUtil.directions(direction)
@@ -101,7 +175,7 @@ object TransferToHFile extends SparkApp {
 
     val ts = System.currentTimeMillis()
     val propsWithTs = Map(LabelMeta.timestamp -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
-    val edge = GraphSubscriberHelper.builder.newEdge(vertex, vertex, label, dir, propsWithTs=propsWithTs)
+    val edge = GraphSubscriberHelper.builder.newEdge(vertex, vertex, label, dir, propsWithTs = propsWithTs)
 
     edge.edgesWithIndex.flatMap { indexEdge =>
       GraphSubscriberHelper.g.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
@@ -115,13 +189,13 @@ object TransferToHFile extends SparkApp {
       (key, value) <- degreeKeyVals
       putRequest <- buildDegreePutRequests(key.vertexIdStr, key.labelName, key.direction, value)
     } yield {
-        val p = putRequest
-        val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
-        kv
-      }
+      val p = putRequest
+      val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+      kv
+    }
     kvs.toIterator
   }
-  
+
   def toKeyValues(strs: Seq[String], labelMapping: Map[String, String], autoEdgeCreate: Boolean): Iterator[KeyValue] = {
     val kvList = new java.util.ArrayList[KeyValue]
     for (s <- strs) {
@@ -143,71 +217,141 @@ object TransferToHFile extends SparkApp {
             val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
             kvList.add(kv)
           }
-        } 
+        }
       }
     }
     kvList.iterator()
   }
-  
-
 
-  override def run() = {
-    val input = args(0)
-    val tmpPath = args(1)
-    val zkQuorum = args(2)
-    val tableName = args(3)
-    val dbUrl = args(4)
-    val maxHFilePerResionServer = args(5).toInt
-    val labelMapping = if (args.length >= 7) GraphSubscriberHelper.toLabelMapping(args(6)) else Map.empty[String, String]
-    val autoEdgeCreate = if (args.length >= 8) args(7).toBoolean else false
-    val buildDegree = if (args.length >= 9) args(8).toBoolean else true
-    val compressionAlgorithm = if (args.length >= 10) args(9) else "lz4"
-    val conf = sparkConf(s"$input: TransferToHFile")
-    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
-    conf.set("spark.kryoserializer.buffer.mb", "24")
+  def generateKeyValues(sc: SparkContext,
+                        s2Config: Config,
+                        input: RDD[String],
+                        graphFileOptions: GraphFileOptions): RDD[KeyValue] = {
+    val kvs = input.mapPartitions { iter =>
+      GraphSubscriberHelper.apply(s2Config)
 
-    val sc = new SparkContext(conf)
-
-    GraphSubscriberHelper.management.createStorageTable(zkQuorum, tableName, List("e", "v"), maxHFilePerResionServer, None, compressionAlgorithm)
-
-    /* set up hbase init */
-    val hbaseConf = HBaseConfiguration.create()
-    hbaseConf.set("hbase.zookeeper.quorum", zkQuorum)
-    hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, tableName)
-    hbaseConf.set("hadoop.tmp.dir", s"/tmp/$tableName")
-
-
-    val rdd = sc.textFile(input)
-
-
-    val kvs = rdd.mapPartitions { iter =>
-      val phase = System.getProperty("phase")
-      GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
-      toKeyValues(iter.toSeq, labelMapping, autoEdgeCreate)
+      toKeyValues(iter.toSeq, graphFileOptions.labelMapping, graphFileOptions.autoEdgeCreate)
     }
 
-    val merged = if (!buildDegree) kvs
+    if (!graphFileOptions.buildDegree) kvs
     else {
-      kvs ++ buildDegrees(rdd, labelMapping, autoEdgeCreate).reduceByKey { (agg, current) =>
+      kvs ++ buildDegrees(input, graphFileOptions.labelMapping, graphFileOptions.autoEdgeCreate).reduceByKey { (agg, current) =>
         agg + current
       }.mapPartitions { iter =>
-        val phase = System.getProperty("phase")
-        GraphSubscriberHelper.apply(phase, dbUrl, "none", "none")
+        GraphSubscriberHelper.apply(s2Config)
+
         toKeyValues(iter.toSeq)
       }
     }
+  }
+
+  def generateHFile(sc: SparkContext,
+                    s2Config: Config,
+                    kvs: RDD[KeyValue],
+                    options: GraphFileOptions): Unit = {
+    val hbaseConfig = toHBaseConfig(options)
+    val startKeys =
+      if (options.incrementalLoad) {
+        // need hbase connection to existing table to figure out the ranges of regions.
+        getTableStartKeys(hbaseConfig, TableName.valueOf(options.tableName))
+      } else {
+        // otherwise we do not need to initialize Connection to hbase cluster.
+        // only numRegions determine region's pre-split.
+        getStartKeys(numRegions = options.numRegions)
+      }
+
+    val hbaseSc = new HBaseContext(sc, hbaseConfig)
 
-    val hbaseSc = new HBaseContext(sc, hbaseConf)
     def flatMap(kv: KeyValue): Iterator[(KeyFamilyQualifier, Array[Byte])] = {
       val k = new KeyFamilyQualifier(CellUtil.cloneRow(kv), CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))
       val v = CellUtil.cloneValue(kv)
       Seq((k -> v)).toIterator
     }
-    val familyOptions = new FamilyHFileWriteOptions(Algorithm.LZ4.getName.toUpperCase,
+
+    val compressionAlgorithmClass = Algorithm.valueOf(options.compressionAlgorithm).getName.toUpperCase
+    val familyOptions = new FamilyHFileWriteOptions(compressionAlgorithmClass,
       BloomType.ROW.name().toUpperCase, 32768, DataBlockEncoding.FAST_DIFF.name().toUpperCase)
     val familyOptionsMap = Map("e".getBytes("UTF-8") -> familyOptions, "v".getBytes("UTF-8") -> familyOptions)
 
-    hbaseSc.bulkLoad(merged, TableName.valueOf(tableName), flatMap, tmpPath, familyOptionsMap)
+
+    hbaseSc.bulkLoad(kvs, TableName.valueOf(options.tableName), startKeys, flatMap, options.tmpPath, familyOptionsMap)
+  }
+
+  def getTableStartKeys(hbaseConfig: Configuration, tableName: TableName): Array[Array[Byte]] = {
+    val conn = ConnectionFactory.createConnection(hbaseConfig)
+    val regionLocator = conn.getRegionLocator(tableName)
+    regionLocator.getStartKeys
+  }
+
+  def toHBaseConfig(graphFileOptions: GraphFileOptions): Configuration = {
+    val hbaseConf = HBaseConfiguration.create()
+
+    hbaseConf.set("hbase.zookeeper.quorum", graphFileOptions.zkQuorum)
+    hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, graphFileOptions.tableName)
+    hbaseConf.set("hadoop.tmp.dir", s"/tmp/${graphFileOptions.tableName}")
+
+    hbaseConf
+  }
+
+  def getStartKeys(numRegions: Int): Array[Array[Byte]] = {
+    val startKey = AsynchbaseStorageManagement.getStartKey(numRegions)
+    val endKey = AsynchbaseStorageManagement.getEndKey(numRegions)
+    if (numRegions < 3) {
+      throw new IllegalArgumentException("Must create at least three regions")
+    } else if (Bytes.compareTo(startKey, endKey) >= 0) {
+      throw new IllegalArgumentException("Start key must be smaller than end key")
+    }
+    val empty = new Array[Byte](0)
+
+    if (numRegions == 3) {
+      Array(empty, startKey, endKey)
+    } else {
+      val splitKeys: Array[Array[Byte]] = Bytes.split(startKey, endKey, numRegions - 3)
+      if (splitKeys == null || splitKeys.length != numRegions - 1) {
+        throw new IllegalArgumentException("Unable to split key range into enough regions")
+      }
+      Array(empty) ++ splitKeys.toSeq
+    }
+  }
+
+  override def run() = {
+    parser.parse(args, GraphFileOptions()) match {
+      case Some(o) => options = o
+      case None =>
+        parser.showUsage()
+        throw new IllegalArgumentException("failed to parse options...")
+    }
+
+    println(s">>> Options: ${options}")
+    val s2Config = Management.toConfig(options.toConfigParams)
+
+    val conf = sparkConf(s"TransferToHFile")
+
+    conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
+    conf.set("spark.kryoserializer.buffer.mb", "24")
+
+    val sc = new SparkContext(conf)
+    val rdd = sc.textFile(options.input)
+
+    GraphSubscriberHelper.apply(s2Config)
+
+    val merged = TransferToHFile.generateKeyValues(sc, s2Config, rdd, options)
+    generateHFile(sc, s2Config, merged, options)
+//    /* set up hbase init */
+//    val hbaseSc = new HBaseContext(sc, toHBaseConfig(options))
+//
+//    def flatMap(kv: KeyValue): Iterator[(KeyFamilyQualifier, Array[Byte])] = {
+//      val k = new KeyFamilyQualifier(CellUtil.cloneRow(kv), CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))
+//      val v = CellUtil.cloneValue(kv)
+//      Seq((k -> v)).toIterator
+//    }
+//
+//    val familyOptions = new FamilyHFileWriteOptions(Algorithm.LZ4.getName.toUpperCase,
+//      BloomType.ROW.name().toUpperCase, 32768, DataBlockEncoding.FAST_DIFF.name().toUpperCase)
+//    val familyOptionsMap = Map("e".getBytes("UTF-8") -> familyOptions, "v".getBytes("UTF-8") -> familyOptions)
+//
+//    val startKeys = getStartKeys(numRegions = options.numRegions)
+//    hbaseSc.bulkLoad(merged, TableName.valueOf(options.tableName), startKeys, flatMap, options.tmpPath, familyOptionsMap)
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
----------------------------------------------------------------------
diff --git a/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
new file mode 100644
index 0000000..6918ce4
--- /dev/null
+++ b/loader/src/test/scala/org/apache/s2graph/loader/subscriber/TransferToHFileTest.scala
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.s2graph.loader.subscriber
+
+import java.io.PrintWriter
+import java.util
+
+import com.typesafe.config.ConfigFactory
+import org.apache.hadoop.hbase.HBaseConfiguration
+import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles
+import org.apache.hadoop.util.ToolRunner
+import org.apache.s2graph.core.{Management, PostProcess}
+import org.apache.s2graph.core.Management.JsonModel.{Index, Prop}
+import org.apache.s2graph.core.mysqls.{Label, ServiceColumn}
+import org.apache.s2graph.core.storage.CanSKeyValue
+import org.apache.s2graph.core.storage.hbase.AsynchbaseStorage
+import org.apache.s2graph.core.types.HBaseType
+import org.apache.s2graph.loader.subscriber.TransferToHFile.options
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, FunSuite, Matchers}
+import play.api.libs.json.Json
+
+import scala.util.Try
+
+class TransferToHFileTest extends FunSuite with Matchers with BeforeAndAfterAll {
+
+  import TransferToHFile._
+  import scala.collection.JavaConverters._
+
+  private val master = "local[2]"
+  private val appName = "example-spark"
+
+  private var sc: SparkContext = _
+
+  /* TransferHFile parameters */
+  val options = GraphFileOptions(
+    input = "/tmp/test.txt",
+    tmpPath = "/tmp/s2graph",
+    zkQuorum = "localhost",
+    dbUrl = "jdbc:h2:file:./var/metastore;MODE=MYSQL",
+    dbUser = "sa",
+    dbPassword = "sa",
+    tableName = "s2graph",
+    maxHFilePerRegionServer = 1,
+    numRegions = 3,
+    compressionAlgorithm = "NONE",
+    buildDegree = false,
+    autoEdgeCreate = false)
+
+  val s2Config = Management.toConfig(options.toConfigParams)
+
+  val tableName = options.tableName
+  val schemaVersion = HBaseType.DEFAULT_VERSION
+  val compressionAlgorithm: String = options.compressionAlgorithm
+
+  override def beforeAll(): Unit = {
+    // initialize spark context.
+    val conf = new SparkConf()
+      .setMaster(master)
+      .setAppName(appName)
+
+    sc = new SparkContext(conf)
+
+    GraphSubscriberHelper.apply(s2Config)
+  }
+
+  override def afterAll(): Unit = {
+    GraphSubscriberHelper.g.shutdown()
+    if (sc != null) {
+      sc.stop()
+    }
+  }
+
+  private def writeToFile(fileName: String)(lines: Seq[String]): Unit = {
+    val writer = new PrintWriter(fileName)
+    lines.foreach(line => writer.write(line + "\n"))
+    writer.close
+  }
+
+  private def initTestEdgeSchema(): Label = {
+    import scala.collection.JavaConverters._
+    /* initialize model for test */
+    val management = GraphSubscriberHelper.management
+
+    val service = management.createService(serviceName = "s2graph", cluster = "localhost",
+      hTableName = "s2graph", preSplitSize = -1, hTableTTL = -1, compressionAlgorithm = "gz")
+
+    val serviceColumn = management.createServiceColumn(service.serviceName, "user", "string", Nil)
+
+    Try {
+      management.createLabel("friends", serviceColumn, serviceColumn, isDirected = true,
+        serviceName = service.serviceName, indices = new java.util.ArrayList[Index],
+        props = Seq(Prop("since", "0", "long"), Prop("score", "0", "integer")).asJava, consistencyLevel = "strong", hTableName = tableName,
+        hTableTTL = -1, schemaVersion = schemaVersion, compressionAlgorithm = compressionAlgorithm, options = "")
+    }
+
+    Label.findByName("friends").getOrElse(throw new IllegalArgumentException("friends label is not initialized."))
+  }
+
+  private def initTestVertexSchema(): ServiceColumn = {
+    import scala.collection.JavaConverters._
+    /* initialize model for test */
+    val management = GraphSubscriberHelper.management
+
+    val service = management.createService(serviceName = "s2graph", cluster = "localhost",
+      hTableName = "s2graph", preSplitSize = -1, hTableTTL = -1, compressionAlgorithm = "gz")
+
+    management.createServiceColumn(service.serviceName, "imei", "string",
+      Seq(
+        Prop(name = "first_time", defaultValue = "''", dataType = "string"),
+        Prop(name = "last_time", defaultValue = "''", dataType = "string"),
+        Prop(name = "total_active_days", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "query_amount", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "active_months", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "fua", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_province", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_city", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_often_days", defaultValue = "-1", dataType = "integer"),
+        Prop(name = "location_last_province", defaultValue = "''", dataType = "string"),
+        Prop(name = "location_last_city", defaultValue = "''", dataType = "string"),
+        Prop(name = "fimei_legality", defaultValue = "-1", dataType = "integer")
+      ))
+  }
+
+  test("test generateKeyValues edge only.") {
+    import scala.collection.JavaConverters._
+    import org.apache.s2graph.core.storage.CanSKeyValue._
+
+    val label = initTestEdgeSchema()
+    /* end of initialize model */
+
+    val bulkEdgeString = "1416236400000\tinsert\tedge\ta\tb\tfriends\t{\"since\":1316236400000,\"score\":10}"
+    val input = sc.parallelize(Seq(bulkEdgeString))
+
+    val kvs = TransferToHFile.generateKeyValues(sc, s2Config, input, options)
+
+    val ls = kvs.map(kv => CanSKeyValue.hbaseKeyValue.toSKeyValue(kv)).collect().toList
+
+    val serDe = GraphSubscriberHelper.g.defaultStorage.serDe
+
+    //    val snapshotEdgeOpt = serDe.indexEdgeDeserializer(label.schemaVersion).fromKeyValues(Seq(ls.head), None)
+    //    val indexEdgeOpt = serDe.indexEdgeDeserializer(label.schemaVersion).fromKeyValues(Seq(ls.last), None)
+
+    val bulkEdge = GraphSubscriberHelper.g.elementBuilder.toGraphElement(bulkEdgeString, options.labelMapping).get
+
+    val indexEdges = ls.flatMap { kv =>
+      serDe.indexEdgeDeserializer(label.schemaVersion).fromKeyValues(Seq(kv), None)
+    }
+
+    val indexEdge = indexEdges.head
+
+    bulkEdge shouldBe(indexEdge)
+  }
+
+
+  test("test generateKeyValues vertex only.") {
+    val serviceColumn = initTestVertexSchema()
+    val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\ts2graph\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广东省\",\"location_often_city\":\"深圳市\",\"location_often_days\":6,\"location_last_province\":\"广东省\",\"location_last_city\":\"深圳市\",\"fimei_legality\":3}"
+    val bulkVertex = GraphSubscriberHelper.g.elementBuilder.toGraphElement(bulkVertexString, options.labelMapping).get
+
+    val input = sc.parallelize(Seq(bulkVertexString))
+
+    val kvs = TransferToHFile.generateKeyValues(sc, s2Config, input, options)
+
+    val ls = kvs.map(kv => CanSKeyValue.hbaseKeyValue.toSKeyValue(kv)).collect().toList
+
+    val serDe = GraphSubscriberHelper.g.defaultStorage.serDe
+
+
+
+    val vertex = serDe.vertexDeserializer(serviceColumn.schemaVersion).fromKeyValues(ls, None).get
+
+    PostProcess.s2VertexToJson(vertex).foreach { jsValue =>
+      println(Json.prettyPrint(jsValue))
+    }
+
+    bulkVertex shouldBe(vertex)
+  }
+
+  test("test generateHFile vertex only.") {
+    val serviceColumn = initTestVertexSchema()
+
+    val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\ts2graph\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广东省\",\"location_often_city\":\"深圳市\",\"location_often_days\":6,\"location_last_province\":\"广东省\",\"location_last_city\":\"深圳市\",\"fimei_legality\":3}"
+    val input = sc.parallelize(Seq(bulkVertexString))
+
+    val kvs = TransferToHFile.generateKeyValues(sc, s2Config, input, options)
+    TransferToHFile.generateHFile(sc, s2Config, kvs, options)
+  }
+
+  test("test loader script.") {
+    val serviceColumn = initTestVertexSchema()
+
+    val bulkVertexString = "20171201\tinsert\tvertex\t800188448586078\ts2graph\timei\t{\"first_time\":\"20171025\",\"last_time\":\"20171112\",\"total_active_days\":14,\"query_amount\":1526.0,\"active_months\":2,\"fua\":\"M5+Note\",\"location_often_province\":\"广东省\",\"location_often_city\":\"深圳市\",\"location_often_days\":6,\"location_last_province\":\"广东省\",\"location_last_city\":\"深圳市\",\"fimei_legality\":3}"
+    val bulkVertexLs = Seq(bulkVertexString)
+    writeToFile(options.input)(bulkVertexLs)
+
+    val input = sc.parallelize(bulkVertexLs)
+    GraphSubscriberHelper.apply(s2Config)
+    val graph = GraphSubscriberHelper.g
+    val vertex = graph.elementBuilder.toVertex(bulkVertexString).get
+
+    val kvs = TransferToHFile.generateKeyValues(sc, s2Config, input, options)
+    TransferToHFile.generateHFile(sc, s2Config, kvs, options)
+
+    val hfileArgs = Array(options.tmpPath, options.tableName)
+    val hbaseConfig = HBaseConfiguration.create()
+
+    val ret = ToolRunner.run(hbaseConfig, new LoadIncrementalHFiles(hbaseConfig), hfileArgs)
+
+    val vertexId = graph.elementBuilder.newVertexId("s2graph")("imei")("800188448586078")
+    val vertexOpt = graph.getVertex(vertexId)
+
+    vertexOpt.isDefined shouldBe(true)
+    vertexOpt.get shouldBe (vertex)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/project/Common.scala
----------------------------------------------------------------------
diff --git a/project/Common.scala b/project/Common.scala
index d3b8d93..e2323a2 100644
--- a/project/Common.scala
+++ b/project/Common.scala
@@ -21,6 +21,7 @@ import sbt._
 
 object Common {
   val sparkVersion = "1.4.1"
+  val spark2Version = "2.3.0"
   val playVersion = "2.5.9"
   val specs2Version = "3.8.5"
 

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
index c8c25b3..0478413 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/GraphElementBuilder.scala
@@ -52,7 +52,7 @@ class GraphElementBuilder(graph: S2GraphLike) {
     element
   } recover {
     case e: Exception =>
-      logger.error(s"[toElement]: $e", e)
+      logger.error(s"[toElement]: $s", e)
       None
   } get
 

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
index 9de3d9d..b92e47b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/JSONParser.scala
@@ -344,8 +344,9 @@ object JSONParser {
   def fromJsonToProperties(jsObject: JsObject): Map[String, Any] = {
     val kvs = for {
       (k, v) <- jsObject.fieldSet
+      anyVal <- jsValueToAny(v)
     } yield {
-        k -> jsValueToString(v)
+        k -> anyVal
       }
     kvs.toMap
   }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
index 819c378..95c3c7b 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/ServiceColumn.scala
@@ -121,11 +121,13 @@ case class ServiceColumn(id: Option[Int],
   lazy val toJson = Json.obj("serviceName" -> service.serviceName, "columnName" -> columnName, "columnType" -> columnType)
 
   def propsToInnerVals(props: Map[String, Any]): Map[ColumnMeta, InnerValLike] = {
-    for {
+    val ret = for {
       (k, v) <- props
       labelMeta <- metasInvMap.get(k)
       innerVal = toInnerVal(v, labelMeta.dataType, schemaVersion)
     } yield labelMeta -> innerVal
+
+    ret
   }
 
   def innerValsToProps(props: Map[Int, InnerValLike]): Map[String, Any] = {

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
index 775afda..57adc8a 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/SKeyValue.scala
@@ -65,6 +65,10 @@ object CanSKeyValue {
     SKeyValue(Array.empty[Byte], kv.key(), kv.family(), kv.qualifier(), kv.value(), kv.timestamp())
   }
 
+  implicit val hbaseKeyValue = instance[org.apache.hadoop.hbase.KeyValue] { kv =>
+    SKeyValue(Array.empty[Byte], kv.getRow, kv.getFamily, kv.getQualifier, kv.getValue, kv.getTimestamp)
+  }
+
   // For asyncbase KeyValues
   implicit val sKeyValue = instance[SKeyValue](identity)
 

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
index 32d640c..78da629 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/StorageSerDe.scala
@@ -70,7 +70,7 @@ trait StorageSerDe {
     **/
   def snapshotEdgeDeserializer(schemaVer: String): Deserializable[SnapshotEdge]
 
-  def indexEdgeDeserializer(schemaVer: String): IndexEdgeDeserializable
+  def indexEdgeDeserializer(schemaVer: String): Deserializable[S2EdgeLike]
 
   def vertexDeserializer(schemaVer: String): Deserializable[S2VertexLike]
 

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageManagement.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageManagement.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageManagement.scala
index 0fb3173..8475ba6 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageManagement.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorageManagement.scala
@@ -56,6 +56,14 @@ object AsynchbaseStorageManagement {
   val DefaultCreateTableOptions = Map(
     "hbase.zookeeper.quorum" -> "localhost"
   )
+
+  def getStartKey(regionCount: Int): Array[Byte] = {
+    Bytes.toBytes((Int.MaxValue / regionCount))
+  }
+
+  def getEndKey(regionCount: Int): Array[Byte] = {
+    Bytes.toBytes((Int.MaxValue / regionCount * (regionCount - 1)))
+  }
 }
 
 class AsynchbaseStorageManagement(val config: Config, val clients: Seq[HBaseClient]) extends StorageManagement {
@@ -271,12 +279,4 @@ class AsynchbaseStorageManagement(val config: Config, val clients: Seq[HBaseClie
       conn.getAdmin
     }
   }
-
-  private def getStartKey(regionCount: Int): Array[Byte] = {
-    Bytes.toBytes((Int.MaxValue / regionCount))
-  }
-
-  private def getEndKey(regionCount: Int): Array[Byte] = {
-    Bytes.toBytes((Int.MaxValue / regionCount * (regionCount - 1)))
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/build.sbt
----------------------------------------------------------------------
diff --git a/s2jobs/build.sbt b/s2jobs/build.sbt
new file mode 100644
index 0000000..b915238
--- /dev/null
+++ b/s2jobs/build.sbt
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Common._
+
+name := "s2jobs"
+
+scalacOptions ++= Seq("-deprecation")
+
+projectDependencies := Seq(
+  (projectID in "s2core").value exclude("org.mortbay.jetty", "j*") exclude("javax.xml.stream", "s*") exclude("javax.servlet", "s*") exclude("javax.servlet", "j*")
+)
+
+libraryDependencies ++= Seq(
+  "com.google.guava" % "guava" % "12.0.1" force(), // use this old version of guava to avoid incompatibility
+  "org.apache.spark" %% "spark-core" % spark2Version,
+  "org.apache.spark" %% "spark-streaming" % spark2Version % "provided",
+  "org.apache.spark" %% "spark-hive" % spark2Version % "provided",
+  "org.specs2" %% "specs2-core" % specs2Version % "test",
+  "org.scalatest" %% "scalatest" % "2.2.1" % "test",
+  "org.apache.hadoop" % "hadoop-distcp" % hadoopVersion,
+  "com.github.scopt" %% "scopt" % "3.7.0"
+)
+
+crossScalaVersions := Seq("2.10.6")
+
+mergeStrategy in assembly := {
+  case PathList("META-INF", ps @ _*) => MergeStrategy.discard
+  case _ => MergeStrategy.first
+}
+
+excludedJars in assembly := {
+  val cp = (fullClasspath in assembly).value
+  cp filter {_.data.getName == "guava-16.0.1.jar"}
+}
+
+test in assembly := {}
+
+parallelExecution in Test := false
+
+mainClass in (Compile) := None

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/loader.py
----------------------------------------------------------------------
diff --git a/s2jobs/loader.py b/s2jobs/loader.py
new file mode 100644
index 0000000..d3a9e67
--- /dev/null
+++ b/s2jobs/loader.py
@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os, sys
+#, urllib2, urllib
+
+def cleanup(args):
+	cmd = "hadoop fs -rm -r /tmp/%s" % args["htable_name"]
+	print(cmd)
+	ret = os.system(cmd)
+	print(cmd, "return", ret)
+	return ret
+
+def hfile(args):
+	print(args)
+	cmd = """HADOOP_CONF_DIR=%s spark-submit --class "org.apache.s2graph.s2jobs.loader.GraphFileGenerator" \
+--name "GraphFileGenerator@shon" \
+--conf "spark.task.maxFailures=20" \
+--conf "spark.executor.extraClassPath=%s" \
+--conf "spark.driver.extraClassPath=%s" \
+--jars %s \
+--master local[2] \
+--num-executors %s \
+--driver-memory 1g \
+--executor-memory 2g \
+--executor-cores 1 \
+%s \
+--input %s \
+--tempDir %s \
+--output /tmp/%s \
+--zkQuorum %s \
+--table %s \
+--dbUrl '%s' \
+--dbUser %s \
+--dbPassword %s \
+--dbDriver %s \
+--maxHFilePerRegionServer %s \
+--labelMapping %s \
+--autoEdgeCreate %s""" % (args["HADOOP_CONF_DIR"],
+						  MYSQL_JAR,
+						  MYSQL_JAR,
+						  MYSQL_JAR,
+						  args["num_executors"],
+						  JAR,
+						  args["input"],
+						  args["tempDir"],
+						  args["htable_name"],
+						  args["hbase_zk"],
+						  args["htable_name"],
+						  args["db_url"],
+						  args["db_user"],
+						  args["db_password"],
+						  args["db_driver"],
+						  args["max_file_per_region"],
+						  args["label_mapping"],
+						  args["auto_create_edge"])
+	print(cmd)
+	ret = os.system(cmd)
+	print(cmd, "return", ret)
+	return ret
+
+def distcp(args):
+	cmd = "hadoop distcp -overwrite -m %s -bandwidth %s /tmp/%s %s/tmp/%s" % (args["-m"], args["-bandwidth"], args["htable_name"], args["hbase_namenode"], args["htable_name"])
+	print(cmd)
+	ret = os.system(cmd)
+	print(cmd, "return", ret)
+	return ret
+
+def chmod(args):
+	cmd = "export HADOOP_CONF_DIR=%s; export HADOOP_USER_NAME=hdfs; hadoop fs -chmod -R 777 /tmp/%s" % (args["HADOOP_CONF_DIR"], args["htable_name"])
+	print(cmd)
+	ret = os.system(cmd)
+	print(cmd, "return", ret)
+	return ret
+
+def load(args):
+	cmd = "export HADOOP_CONF_DIR=%s; export HBASE_CONF_DIR=%s; hbase %s /tmp/%s %s" % \
+		  (args["HADOOP_CONF_DIR"], args["HBASE_CONF_DIR"], LOADER_CLASS, args["htable_name"], args["htable_name"])
+	print(cmd)
+	ret = os.system(cmd)
+	print(cmd, "return", ret)
+	return ret
+
+def send(msg):
+	print(msg)
+
+def run(args):
+	cleanup(args)
+	send("[Start]: bulk loader")
+	ret = hfile(args)
+
+	if ret != 0: return send("[Failed]: loader build hfile failed %s" % ret)
+	else: send("[Success]: loader build hfile")
+
+	# ret = distcp(args)
+	#
+	# if ret != 0: return send("[Failed]: loader distcp failed %s" % ret)
+	# else: send("[Success]: loader distcp")
+	#
+	# ret = chmod(args)
+	#
+	# if ret != 0: return send("[Failed]: loader chmod failed %s" % ret)
+	# else: send("[Success]: loader chmod")
+
+	ret = load(args)
+
+	if ret != 0: return send("[Failed]: loader complete bulkload failed %s" % ret)
+	else: send("[Success]: loader complete bulkload")
+
+
+LOADER_CLASS = "org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles"
+JAR="s2jobs/target/scala-2.11/s2jobs-assembly-0.2.1-SNAPSHOT.jar"
+MYSQL_JAR="/Users/shon/Downloads/mysql-connector-java-5.1.28.jar"
+MYSQL_CLASSPATH="/Users/shon/Downloads/mysql-connector-java-5.1.28.jar"
+DB_DRIVER="com.mysql.jdbc.Driver"
+DB_URL="jdbc:mysql://localhost:3306/graph_dev"
+# DB_URL="jdbc:h2:file:./var/metastore;MODE=MYSQL"
+args = {
+	"HADOOP_CONF_DIR": "/usr/local/Cellar/hadoop/2.7.3/libexec/etc/hadoop",
+	"HBASE_CONF_DIR": "/usr/local/opt/hbase/libexec/conf",
+	"htable_name": "test",
+	"hbase_namenode": "hdfs://localhost:8020",
+	"hbase_zk": "localhost",
+	"db_driver": DB_DRIVER,
+	"db_url": DB_URL,
+	"db_user": "graph",
+	"db_password": "graph",
+	"max_file_per_region": 1,
+	"label_mapping": "none",
+	"auto_create_edge": "false",
+	"-m": 1,
+	"-bandwidth": 10,
+	"num_executors": 2,
+	"input": "/tmp/imei-20.txt",
+	"tempDir": "/tmp/bulkload_tmp"
+}
+
+run(args)

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/java/org/apache/hadoop/hbase/mapreduce/GraphHFileOutputFormat.java
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/java/org/apache/hadoop/hbase/mapreduce/GraphHFileOutputFormat.java b/s2jobs/src/main/java/org/apache/hadoop/hbase/mapreduce/GraphHFileOutputFormat.java
new file mode 100644
index 0000000..4d6599f
--- /dev/null
+++ b/s2jobs/src/main/java/org/apache/hadoop/hbase/mapreduce/GraphHFileOutputFormat.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+
+public class GraphHFileOutputFormat extends HFileOutputFormat2 {
+    private static final String COMPRESSION_FAMILIES_CONF_KEY =
+            "hbase.hfileoutputformat.families.compression";
+    private static final String BLOOM_TYPE_FAMILIES_CONF_KEY =
+            "hbase.hfileoutputformat.families.bloomtype";
+    private static final String BLOCK_SIZE_FAMILIES_CONF_KEY =
+            "hbase.mapreduce.hfileoutputformat.blocksize";
+    private static final String DATABLOCK_ENCODING_FAMILIES_CONF_KEY =
+            "hbase.mapreduce.hfileoutputformat.families.datablock.encoding";
+
+    // This constant is public since the client can modify this when setting
+    // up their conf object and thus refer to this symbol.
+    // It is present for backwards compatibility reasons. Use it only to
+    // override the auto-detection of datablock encoding.
+    public static final String DATABLOCK_ENCODING_OVERRIDE_CONF_KEY =
+            "hbase.mapreduce.hfileoutputformat.datablock.encoding";
+
+    static Log LOG = LogFactory.getLog(GraphHFileOutputFormat.class);
+
+    public static void configureIncrementalLoad(Job job, List<ImmutableBytesWritable> startKeys,
+                                                List<String> familyNames, Compression.Algorithm compression, BloomType bloomType,
+                                                int blockSize, DataBlockEncoding dataBlockEncoding) throws IOException {
+
+        Configuration conf = job.getConfiguration();
+
+        job.setOutputKeyClass(ImmutableBytesWritable.class);
+        job.setOutputValueClass(KeyValue.class);
+        job.setOutputFormatClass(HFileOutputFormat2.class);
+
+        // Based on the configured map output class, set the correct reducer to properly
+        // sort the incoming values.
+        // TODO it would be nice to pick one or the other of these formats.
+        if (KeyValue.class.equals(job.getMapOutputValueClass())) {
+            job.setReducerClass(KeyValueSortReducer.class);
+        } else if (Put.class.equals(job.getMapOutputValueClass())) {
+            job.setReducerClass(PutSortReducer.class);
+        } else if (Text.class.equals(job.getMapOutputValueClass())) {
+            job.setReducerClass(TextSortReducer.class);
+        } else {
+            LOG.warn("Unknown map output value type:" + job.getMapOutputValueClass());
+        }
+
+        conf.setStrings("io.serializations", conf.get("io.serializations"),
+                MutationSerialization.class.getName(), ResultSerialization.class.getName(),
+                KeyValueSerialization.class.getName());
+
+        job.setNumReduceTasks(startKeys.size());
+
+        configurePartitioner(job, startKeys);
+        // Set compression algorithms based on column families
+        configureCompression(familyNames, compression, conf);
+        configureBloomType(familyNames, bloomType, conf);
+        configureBlockSize(familyNames, blockSize, conf);
+        configureDataBlockEncoding(familyNames, dataBlockEncoding, conf);
+
+        TableMapReduceUtil.addDependencyJars(job);
+        TableMapReduceUtil.initCredentials(job);
+        LOG.info("Incremental table output configured.");
+    }
+
+    static void configureCompression(List<String> familyNames, Compression.Algorithm compression,
+                                     Configuration conf) throws IOException {
+        StringBuilder compressionConfigValue = new StringBuilder();
+        int i = 0;
+        for (String familyName : familyNames) {
+            if (i++ > 0) {
+                compressionConfigValue.append('&');
+            }
+            compressionConfigValue.append(URLEncoder.encode(familyName, "UTF-8"));
+            compressionConfigValue.append('=');
+            compressionConfigValue.append(URLEncoder.encode(compression.getName(), "UTF-8"));
+        }
+        // Get rid of the last ampersand
+        conf.set(COMPRESSION_FAMILIES_CONF_KEY, compressionConfigValue.toString());
+    }
+
+    static void configureBloomType(List<String> familyNames, BloomType bloomType, Configuration conf)
+            throws IOException {
+        StringBuilder bloomTypeConfigValue = new StringBuilder();
+        int i = 0;
+        for (String familyName : familyNames) {
+            if (i++ > 0) {
+                bloomTypeConfigValue.append('&');
+            }
+            bloomTypeConfigValue.append(URLEncoder.encode(familyName, "UTF-8"));
+            bloomTypeConfigValue.append('=');
+            String bloomTypeStr = bloomType.toString();
+            if (bloomTypeStr == null) {
+                bloomTypeStr = HColumnDescriptor.DEFAULT_BLOOMFILTER;
+            }
+            bloomTypeConfigValue.append(URLEncoder.encode(bloomTypeStr, "UTF-8"));
+        }
+        conf.set(BLOOM_TYPE_FAMILIES_CONF_KEY, bloomTypeConfigValue.toString());
+    }
+
+    static void configureBlockSize(List<String> familyNames, int blockSize, Configuration conf)
+            throws IOException {
+        StringBuilder blockSizeConfigValue = new StringBuilder();
+        int i = 0;
+        for (String familyName : familyNames) {
+            if (i++ > 0) {
+                blockSizeConfigValue.append('&');
+            }
+            blockSizeConfigValue.append(URLEncoder.encode(familyName, "UTF-8"));
+            blockSizeConfigValue.append('=');
+            blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(blockSize), "UTF-8"));
+        }
+        // Get rid of the last ampersand
+        conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString());
+    }
+
+    static void configureDataBlockEncoding(List<String> familyNames, DataBlockEncoding encoding,
+                                           Configuration conf) throws IOException {
+        StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
+        int i = 0;
+        for (String familyName : familyNames) {
+            if (i++ > 0) {
+                dataBlockEncodingConfigValue.append('&');
+            }
+            dataBlockEncodingConfigValue.append(URLEncoder.encode(familyName, "UTF-8"));
+            dataBlockEncodingConfigValue.append('=');
+            if (encoding == null) {
+                encoding = DataBlockEncoding.NONE;
+            }
+            dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(), "UTF-8"));
+        }
+        conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY, dataBlockEncodingConfigValue.toString());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/S2GraphHelper.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/S2GraphHelper.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/S2GraphHelper.scala
new file mode 100644
index 0000000..ef76608
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/S2GraphHelper.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core._
+
+import scala.concurrent.ExecutionContext
+
+object S2GraphHelper {
+  def initS2Graph(config: Config)(implicit ec: ExecutionContext = ExecutionContext.Implicits.global): S2Graph = {
+    new S2Graph(config)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
new file mode 100644
index 0000000..51476c1
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+import org.apache.s2graph.core._
+import org.apache.spark.{SparkConf, SparkContext}
+
+object GraphFileGenerator {
+  def main(args: Array[String]): Unit = {
+    val options = GraphFileOptions.toOption(args)
+
+    val s2Config = Management.toConfig(options.toConfigParams)
+
+    val conf = new SparkConf()
+    conf.setAppName(this.getClass.getSimpleName)
+    val sc = new SparkContext(conf)
+
+    val input = sc.textFile(options.input)
+    options.method match {
+      case "MR" => HFileMRGenerator.generate(sc, s2Config, input, options)
+      case "SPARK" => HFileGenerator.generate(sc, s2Config, input, options)
+      case _ => throw new IllegalArgumentException("only supported type is MR/SPARK.")
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileOptions.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileOptions.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileOptions.scala
new file mode 100644
index 0000000..3e3ffb9
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileOptions.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+object GraphFileOptions {
+  val parser = new scopt.OptionParser[GraphFileOptions]("run") {
+
+    opt[String]('i', "input").required().action( (x, c) =>
+      c.copy(input = x) ).text("hdfs path for tsv file(bulk format)")
+
+    opt[String]('t', "tempDir").required().action( (x, c) =>
+      c.copy(tempDir = x) ).text("temp hdfs path for staging HFiles")
+
+    opt[String]('o', "output").required().action( (x, c) =>
+      c.copy(output = x) ).text("output hdfs path for storing HFiles")
+
+    opt[String]('z', "zkQuorum").required().action( (x, c) =>
+      c.copy(zkQuorum = x) ).text("zookeeper config for hbase")
+
+    opt[String]('t', "table").required().action( (x, c) =>
+      c.copy(tableName = x) ).text("table name for this bulk upload.")
+
+    opt[String]('c', "dbUrl").required().action( (x, c) =>
+      c.copy(dbUrl = x)).text("jdbc connection url.")
+
+    opt[String]('u', "dbUser").required().action( (x, c) =>
+      c.copy(dbUser = x)).text("database user name.")
+
+    opt[String]('p', "dbPassword").required().action( (x, c) =>
+      c.copy(dbPassword = x)).text("database password.")
+
+    opt[String]('r', "dbDriver").action( (x, c) =>
+      c.copy(dbDriver = x)).text("jdbc driver class.")
+
+    opt[Int]('h', "maxHFilePerRegionServer").action ( (x, c) =>
+      c.copy(maxHFilePerRegionServer = x)).text("maximum number of HFile per RegionServer."
+    )
+
+    opt[Int]('n', "numRegions").action ( (x, c) =>
+      c.copy(numRegions = x)).text("total numRegions(pre-split size) on table."
+    )
+
+    opt[String]('l', "labelMapping").action( (x, c) =>
+      c.copy(labelMapping = toLabelMapping(x)) ).text("mapping info to change the label from source (originalLabel:newLabel)")
+
+    opt[Boolean]('d', "buildDegree").action( (x, c) =>
+      c.copy(buildDegree = x)).text("generate degree values")
+
+    opt[Boolean]('a', "autoEdgeCreate").action( (x, c) =>
+      c.copy(autoEdgeCreate = x)).text("generate reverse edge automatically")
+
+    opt[Boolean]('c', "incrementalLoad").action( (x, c) =>
+      c.copy(incrementalLoad = x)).text("whether incremental bulkload which append data on existing table or not."
+    )
+    opt[String]('m', "method").action( (x, c) =>
+      c.copy(method = x)).text("run method. currently MR(default)/SPARK supported."
+    )
+  }
+
+  def toOption(args: Array[String]): GraphFileOptions = {
+    parser.parse(args, GraphFileOptions()) match {
+      case Some(o) => o
+      case None =>
+        parser.showUsage()
+        throw new IllegalArgumentException("failed to parse options...")
+    }
+  }
+
+  private def toLabelMapping(lableMapping: String): Map[String, String] = {
+    (for {
+      token <- lableMapping.split(",")
+      inner = token.split(":") if inner.length == 2
+    } yield {
+      (inner.head, inner.last)
+    }).toMap
+  }
+}
+/**
+  * Option case class for TransferToHFile.
+  * @param input
+  * @param output
+  * @param zkQuorum
+  * @param tableName
+  * @param dbUrl
+  * @param dbUser
+  * @param dbPassword
+  * @param maxHFilePerRegionServer
+  * @param numRegions
+  * @param labelMapping
+  * @param autoEdgeCreate
+  * @param buildDegree
+  * @param incrementalLoad
+  * @param compressionAlgorithm
+  */
+case class GraphFileOptions(input: String = "",
+                            tempDir: String = "",
+                            output: String = s"/tmp/bulkload",
+                            zkQuorum: String = "",
+                            tableName: String = "",
+                            dbUrl: String = "",
+                            dbUser: String = "",
+                            dbPassword: String = "",
+                            dbDriver: String = "org.h2.Driver",
+                            maxHFilePerRegionServer: Int = 1,
+                            numRegions: Int = 3,
+                            labelMapping: Map[String, String] = Map.empty[String, String],
+                            autoEdgeCreate: Boolean = false,
+                            buildDegree: Boolean = false,
+                            incrementalLoad: Boolean = false,
+                            compressionAlgorithm: String = "NONE",
+                            method: String = "SPARK") {
+  def toConfigParams = {
+    Map(
+      "hbase.zookeeper.quorum" -> zkQuorum,
+      "db.default.url" -> dbUrl,
+      "db.default.user" -> dbUser,
+      "db.default.password" -> dbPassword,
+      "db.default.driver" -> dbDriver
+    )
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
new file mode 100644
index 0000000..9c3de2a
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+import com.typesafe.config.Config
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.client.ConnectionFactory
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
+import org.apache.hadoop.hbase.regionserver.BloomType
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.hbase.{CellUtil, HBaseConfiguration, KeyValue, TableName}
+import org.apache.s2graph.core._
+import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
+import org.apache.s2graph.core.storage.hbase.AsynchbaseStorageManagement
+import org.apache.s2graph.core.types.{InnerValLikeWithTs, SourceVertexId}
+import org.apache.s2graph.s2jobs.S2GraphHelper
+import org.apache.s2graph.s2jobs.spark._
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.hbase.async.PutRequest
+import play.api.libs.json.Json
+
+object HFileGenerator extends RawFileGenerator {
+
+  import scala.collection.JavaConverters._
+
+  private def insertBulkForLoaderAsync(s2: S2Graph, edge: S2Edge, createRelEdges: Boolean = true): List[PutRequest] = {
+    val relEdges = if (createRelEdges) edge.relatedEdges else List(edge)
+
+    buildPutRequests(s2, edge.toSnapshotEdge) ++ relEdges.toList.flatMap { e =>
+      e.edgesWithIndex.flatMap { indexEdge => buildPutRequests(s2, indexEdge) }
+    }
+  }
+
+  def buildPutRequests(s2: S2Graph, snapshotEdge: SnapshotEdge): List[PutRequest] = {
+    val kvs = s2.getStorage(snapshotEdge.label).serDe.snapshotEdgeSerializer(snapshotEdge).toKeyValues.toList
+    kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
+  }
+
+  def buildPutRequests(s2: S2Graph, indexEdge: IndexEdge): List[PutRequest] = {
+    val kvs = s2.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.toList
+    kvs.map { kv => new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp) }
+  }
+
+  def buildDegreePutRequests(s2: S2Graph, vertexId: String, labelName: String, direction: String, degreeVal: Long): List[PutRequest] = {
+    val label = Label.findByName(labelName).getOrElse(throw new RuntimeException(s"$labelName is not found in DB."))
+    val dir = GraphUtil.directions(direction)
+    val innerVal = JSONParser.jsValueToInnerVal(Json.toJson(vertexId), label.srcColumnWithDir(dir).columnType, label.schemaVersion).getOrElse {
+      throw new RuntimeException(s"$vertexId can not be converted into innerval")
+    }
+    val vertex = s2.elementBuilder.newVertex(SourceVertexId(label.srcColumn, innerVal))
+
+    val ts = System.currentTimeMillis()
+    val propsWithTs = Map(LabelMeta.timestamp -> InnerValLikeWithTs.withLong(ts, ts, label.schemaVersion))
+    val edge = s2.elementBuilder.newEdge(vertex, vertex, label, dir, propsWithTs = propsWithTs)
+
+    edge.edgesWithIndex.flatMap { indexEdge =>
+      s2.getStorage(indexEdge.label).serDe.indexEdgeSerializer(indexEdge).toKeyValues.map { kv =>
+        new PutRequest(kv.table, kv.row, kv.cf, Array.empty[Byte], Bytes.toBytes(degreeVal), kv.timestamp)
+      }
+    }
+  }
+
+  def toKeyValues(s2: S2Graph, degreeKeyVals: Seq[(DegreeKey, Long)]): Iterator[KeyValue] = {
+    val kvs = for {
+      (key, value) <- degreeKeyVals
+      putRequest <- buildDegreePutRequests(s2, key.vertexIdStr, key.labelName, key.direction, value)
+    } yield {
+      val p = putRequest
+      val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+      kv
+    }
+    kvs.toIterator
+  }
+
+  def toKeyValues(s2: S2Graph, strs: Seq[String], labelMapping: Map[String, String], autoEdgeCreate: Boolean): Iterator[KeyValue] = {
+    val kvList = new java.util.ArrayList[KeyValue]
+    for (s <- strs) {
+      val elementList = s2.elementBuilder.toGraphElement(s, labelMapping).toSeq
+      for (element <- elementList) {
+        if (element.isInstanceOf[S2Edge]) {
+          val edge = element.asInstanceOf[S2Edge]
+          val putRequestList = insertBulkForLoaderAsync(s2, edge, autoEdgeCreate)
+          for (p <- putRequestList) {
+            val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+            kvList.add(kv)
+          }
+        } else if (element.isInstanceOf[S2Vertex]) {
+          val vertex = element.asInstanceOf[S2Vertex]
+          val putRequestList = s2.getStorage(vertex.service).serDe.vertexSerializer(vertex).toKeyValues.map { kv =>
+            new PutRequest(kv.table, kv.row, kv.cf, kv.qualifier, kv.value, kv.timestamp)
+          }
+          for (p <- putRequestList) {
+            val kv = new KeyValue(p.key(), p.family(), p.qualifier, p.timestamp, p.value)
+            kvList.add(kv)
+          }
+        }
+      }
+    }
+    kvList.iterator().asScala
+  }
+
+
+  def getTableStartKeys(hbaseConfig: Configuration, tableName: TableName): Array[Array[Byte]] = {
+    val conn = ConnectionFactory.createConnection(hbaseConfig)
+    val regionLocator = conn.getRegionLocator(tableName)
+    regionLocator.getStartKeys
+  }
+
+  def toHBaseConfig(graphFileOptions: GraphFileOptions): Configuration = {
+    val hbaseConf = HBaseConfiguration.create()
+
+    hbaseConf.set("hbase.zookeeper.quorum", graphFileOptions.zkQuorum)
+    hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, graphFileOptions.tableName)
+    hbaseConf.set("hadoop.tmp.dir", s"/tmp/${graphFileOptions.tableName}")
+
+    hbaseConf
+  }
+
+  def getStartKeys(numRegions: Int): Array[Array[Byte]] = {
+    val startKey = AsynchbaseStorageManagement.getStartKey(numRegions)
+    val endKey = AsynchbaseStorageManagement.getEndKey(numRegions)
+    if (numRegions < 3) {
+      throw new IllegalArgumentException("Must create at least three regions")
+    } else if (Bytes.compareTo(startKey, endKey) >= 0) {
+      throw new IllegalArgumentException("Start key must be smaller than end key")
+    }
+    val empty = new Array[Byte](0)
+
+    if (numRegions == 3) {
+      Array(empty, startKey, endKey)
+    } else {
+      val splitKeys: Array[Array[Byte]] = Bytes.split(startKey, endKey, numRegions - 3)
+      if (splitKeys == null || splitKeys.length != numRegions - 1) {
+        throw new IllegalArgumentException("Unable to split key range into enough regions")
+      }
+      Array(empty) ++ splitKeys.toSeq
+    }
+  }
+
+  def transfer(sc: SparkContext,
+               s2Config: Config,
+               input: RDD[String],
+               graphFileOptions: GraphFileOptions): RDD[KeyValue] = {
+    val kvs = input.mapPartitions { iter =>
+      val s2 = S2GraphHelper.initS2Graph(s2Config)
+
+      val s = toKeyValues(s2, iter.toSeq, graphFileOptions.labelMapping, graphFileOptions.autoEdgeCreate)
+      s
+    }
+
+    if (!graphFileOptions.buildDegree) kvs
+    else {
+      kvs ++ buildDegrees(input, graphFileOptions.labelMapping, graphFileOptions.autoEdgeCreate).reduceByKey { (agg, current) =>
+        agg + current
+      }.mapPartitions { iter =>
+        val s2 = S2GraphHelper.initS2Graph(s2Config)
+
+        toKeyValues(s2, iter.toSeq)
+      }
+    }
+  }
+
+  def generateHFile(sc: SparkContext,
+                    s2Config: Config,
+                    kvs: RDD[KeyValue],
+                    options: GraphFileOptions): Unit = {
+    val hbaseConfig = toHBaseConfig(options)
+    val startKeys =
+      if (options.incrementalLoad) {
+        // need hbase connection to existing table to figure out the ranges of regions.
+        getTableStartKeys(hbaseConfig, TableName.valueOf(options.tableName))
+      } else {
+        // otherwise we do not need to initialize Connection to hbase cluster.
+        // only numRegions determine region's pre-split.
+        getStartKeys(numRegions = options.numRegions)
+      }
+
+    val hbaseSc = new HBaseContext(sc, hbaseConfig)
+
+    def flatMap(kv: KeyValue): Iterator[(KeyFamilyQualifier, Array[Byte])] = {
+      val k = new KeyFamilyQualifier(CellUtil.cloneRow(kv), CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))
+      val v = CellUtil.cloneValue(kv)
+      Seq((k -> v)).toIterator
+    }
+
+    val compressionAlgorithmClass = Algorithm.valueOf(options.compressionAlgorithm).getName.toUpperCase
+    val familyOptions = new FamilyHFileWriteOptions(compressionAlgorithmClass,
+      BloomType.ROW.name().toUpperCase, 32768, DataBlockEncoding.FAST_DIFF.name().toUpperCase)
+    val familyOptionsMap = Map("e".getBytes("UTF-8") -> familyOptions, "v".getBytes("UTF-8") -> familyOptions)
+
+
+    hbaseSc.bulkLoad(kvs, TableName.valueOf(options.tableName), startKeys, flatMap, options.output, familyOptionsMap.asJava)
+  }
+
+  override def generate(sc: SparkContext,
+                        config: Config,
+                        rdd: RDD[String],
+                        _options: GraphFileOptions): Unit = {
+    val kvs = transfer(sc, config, rdd, _options)
+    generateHFile(sc, config, kvs, _options)
+  }
+}


[2/5] incubator-s2graph git commit: - add s2jobs subproject. - migrate bulk loader with spark 2.3.0. - add test cases for bulk loader.

Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileMRGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileMRGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileMRGenerator.scala
new file mode 100644
index 0000000..ee4c338
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileMRGenerator.scala
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+import java.util.UUID
+
+import com.typesafe.config.Config
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.hbase.{HConstants, KeyValue}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.io.compress.Compression
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.mapreduce.{GraphHFileOutputFormat, HFileOutputFormat2}
+import org.apache.hadoop.hbase.regionserver.BloomType
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, SequenceFileInputFormat}
+import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat, SequenceFileOutputFormat}
+import org.apache.hadoop.mapreduce.{Job, Mapper}
+import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
+import org.apache.s2graph.core.mysqls.Label
+import org.apache.s2graph.core.storage.hbase.AsynchbaseStorageManagement
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+
+object HFileMRGenerator extends RawFileGenerator {
+  val DefaultBlockSize = 32768
+  val DefaultConfig = Map(
+    "yarn.app.mapreduce.am.resource.mb" -> 4096,
+    "mapred.child.java.opts" -> "=-Xmx8g",
+    "mapreduce.job.running.map.limit" -> 200,
+    "mapreduce.job.running.reduce.limit" -> 200,
+    "mapreduce.reduce.memory.mb" -> 8192,
+    "mapreduce.reduce.java.opts" -> "-Xmx6144m",
+    "mapreduce.map.memory.mb" -> "4096",
+    "mapreduce.map.java.opts" -> "-Xmx3072m",
+    "mapreduce.task.io.sort.factor" -> 10
+  )
+
+  class HFileMapper extends Mapper[ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable, KeyValue] {
+    override def map(key: ImmutableBytesWritable, value: ImmutableBytesWritable,
+                     context: Mapper[ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable, KeyValue]#Context): Unit = {
+      val keyValue = new KeyValue(value.get(), 0, value.get().length)
+      context.write(new ImmutableBytesWritable(keyValue.getRow), keyValue)
+    }
+  }
+
+  def setMROptions(conf: Configuration) = {
+    DefaultConfig.foreach { case (k, v) =>
+      conf.set(k, conf.get(k, v.toString))
+    }
+  }
+
+  def getStartKeys(numRegions: Int): Seq[ImmutableBytesWritable] = {
+    val startKey = AsynchbaseStorageManagement.getStartKey(numRegions)
+    val endKey = AsynchbaseStorageManagement.getEndKey(numRegions)
+    if (numRegions < 3) {
+      throw new IllegalArgumentException("Must create at least three regions")
+    }
+    else if (Bytes.compareTo(startKey, endKey) >= 0) {
+      throw new IllegalArgumentException("Start key must be smaller than end key")
+    }
+    val empty = new Array[Byte](0)
+    val results = if (numRegions == 3) {
+      Seq(empty, startKey, endKey)
+    } else {
+      val splitKeys: Array[Array[Byte]] = Bytes.split(startKey, endKey, numRegions - 3)
+      if (splitKeys == null || splitKeys.length != numRegions - 1) {
+        throw new IllegalArgumentException("Unable to split key range into enough regions")
+      }
+      Seq(empty) ++ splitKeys.toSeq
+    }
+    results.map(new ImmutableBytesWritable(_))
+  }
+
+  def sortKeyValues(hbaseConf: Configuration,
+                    tableName: String,
+                    input: String,
+                    output: String,
+                    partitionSize: Int,
+                    compressionAlgorithm: String) = {
+    val job = Job.getInstance(hbaseConf, s"sort-MR $tableName")
+    job.setJarByClass(getClass)
+    job.setInputFormatClass(classOf[SequenceFileInputFormat[ImmutableBytesWritable, ImmutableBytesWritable]])
+    job.setMapperClass(classOf[HFileMapper])
+    job.setOutputKeyClass(classOf[ImmutableBytesWritable])
+    job.setOutputValueClass(classOf[KeyValue])
+    job.setOutputFormatClass(classOf[HFileOutputFormat2])
+    job.getConfiguration.set(HConstants.TEMPORARY_FS_DIRECTORY_KEY, s"/tmp/${tableName}_staging")
+    FileInputFormat.addInputPaths(job, input)
+    FileOutputFormat.setOutputPath(job, new Path(output))
+
+
+    import scala.collection.JavaConversions._
+
+    GraphHFileOutputFormat.configureIncrementalLoad(job,
+      getStartKeys(partitionSize),
+      Seq("e", "v"),
+      Compression.getCompressionAlgorithmByName(compressionAlgorithm.toLowerCase),
+      BloomType.ROW,
+      DefaultBlockSize,
+      DataBlockEncoding.FAST_DIFF
+    )
+    job
+  }
+
+  def transfer(sc: SparkContext,
+               s2Config: Config,
+               input: RDD[String],
+               graphFileOptions: GraphFileOptions): RDD[KeyValue] = {
+    HFileGenerator.transfer(sc, s2Config, input, graphFileOptions)
+  }
+
+  override def generate(sc: SparkContext,
+                        config: Config,
+                        rdd: RDD[String],
+                        options: GraphFileOptions) = {
+    val merged = transfer(sc, config, rdd, options)
+    val tmpOutput = options.tempDir
+
+    merged.map(x => (new ImmutableBytesWritable(x.getKey), new ImmutableBytesWritable(x.getBuffer)))
+      .saveAsNewAPIHadoopFile(tmpOutput,
+        classOf[ImmutableBytesWritable],
+        classOf[ImmutableBytesWritable],
+        classOf[SequenceFileOutputFormat[ImmutableBytesWritable, ImmutableBytesWritable]])
+
+    sc.killExecutors((1 to sc.defaultParallelism).map(_.toString))
+
+    val conf = sc.hadoopConfiguration
+
+    conf.setClassLoader(Thread.currentThread().getContextClassLoader())
+    setMROptions(conf)
+
+    val job = sortKeyValues(conf, options.tableName, tmpOutput, options.output,
+      options.numRegions, options.compressionAlgorithm)
+
+    val success = job.waitForCompletion(true)
+    FileSystem.get(conf).delete(new Path(tmpOutput), true)
+    if (!success) {
+      throw new RuntimeException("mapreduce failed")
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/RawFileGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/RawFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/RawFileGenerator.scala
new file mode 100644
index 0000000..1613f20
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/RawFileGenerator.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs.loader
+
+import com.typesafe.config.Config
+import org.apache.s2graph.core.GraphUtil
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+
+case class DegreeKey(vertexIdStr: String, labelName: String, direction: String)
+
+trait RawFileGenerator {
+  def generate(sc: SparkContext,
+               config:Config,
+               rdd: RDD[String],
+               _options:GraphFileOptions)
+
+  def buildDegrees(msgs: RDD[String], labelMapping: Map[String, String], edgeAutoCreate: Boolean) = {
+    for {
+      msg <- msgs
+      tokens = GraphUtil.split(msg)
+      if tokens(2) == "e" || tokens(2) == "edge"
+      tempDirection = if (tokens.length == 7) "out" else tokens(7)
+      direction = if (tempDirection != "out" && tempDirection != "in") "out" else tempDirection
+      reverseDirection = if (direction == "out") "in" else "out"
+      convertedLabelName = labelMapping.get(tokens(5)).getOrElse(tokens(5))
+      (vertexIdStr, vertexIdStrReversed) = (tokens(3), tokens(4))
+      degreeKey = DegreeKey(vertexIdStr, convertedLabelName, direction)
+      degreeKeyReversed = DegreeKey(vertexIdStrReversed, convertedLabelName, reverseDirection)
+      extra = if (edgeAutoCreate) List(degreeKeyReversed -> 1L) else Nil
+      output <- List(degreeKey -> 1L) ++ extra
+    } yield output
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/BulkLoadPartitioner.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/BulkLoadPartitioner.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/BulkLoadPartitioner.scala
new file mode 100644
index 0000000..4ecce80
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/BulkLoadPartitioner.scala
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import java.util
+import java.util.Comparator
+
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.spark.Partitioner
+
+/**
+ * A Partitioner implementation that will separate records to different
+ * HBase Regions based on region splits
+ *
+ * @param startKeys   The start keys for the given table
+ */
+class BulkLoadPartitioner(startKeys:Array[Array[Byte]])
+  extends Partitioner {
+
+  override def numPartitions: Int = startKeys.length
+
+  override def getPartition(key: Any): Int = {
+
+    val rowKey:Array[Byte] =
+      key match {
+        case qualifier: KeyFamilyQualifier =>
+          qualifier.rowKey
+        case _ =>
+          key.asInstanceOf[Array[Byte]]
+      }
+
+    val comparator: Comparator[Array[Byte]] = new Comparator[Array[Byte]] {
+      override def compare(o1: Array[Byte], o2: Array[Byte]): Int = {
+        Bytes.compareTo(o1, o2)
+      }
+    }
+    val partition = util.Arrays.binarySearch(startKeys, rowKey, comparator)
+    if (partition < 0) partition * -1 + -2
+    else partition
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/FamilyHFileWriteOptions.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/FamilyHFileWriteOptions.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/FamilyHFileWriteOptions.scala
new file mode 100644
index 0000000..ea69c53
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/FamilyHFileWriteOptions.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import java.io.Serializable
+
+/**
+ * This object will hold optional data for how a given column family's
+ * writer will work
+ *
+ * @param compression       String to define the Compression to be used in the HFile
+ * @param bloomType         String to define the bloom type to be used in the HFile
+ * @param blockSize         The block size to be used in the HFile
+ * @param dataBlockEncoding String to define the data block encoding to be used
+ *                          in the HFile
+ */
+class FamilyHFileWriteOptions( val compression:String,
+                               val bloomType: String,
+                               val blockSize: Int,
+                               val dataBlockEncoding: String) extends Serializable

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseContext.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseContext.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseContext.scala
new file mode 100644
index 0000000..d6f6708
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseContext.scala
@@ -0,0 +1,850 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import java.io._
+import java.net.InetSocketAddress
+import java.util
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.hbase._
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.fs.HFileSystem
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.io.compress.Compression
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding
+import org.apache.hadoop.hbase.io.hfile.{CacheConfig, HFileContextBuilder}
+import org.apache.hadoop.hbase.mapreduce.{IdentityTableMapper, TableInputFormat, TableMapReduceUtil}
+import org.apache.hadoop.hbase.regionserver.{BloomType, HStore, StoreFile}
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.mapreduce.Job
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod
+import org.apache.s2graph.s2jobs.spark.HBaseRDDFunctions._
+import org.apache.spark.broadcast.Broadcast
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.rdd.RDD
+import org.apache.spark.streaming.dstream.DStream
+import org.apache.spark.{SerializableWritable, SparkContext}
+
+import scala.collection.mutable
+import scala.reflect.ClassTag
+
+/**
+  * HBaseContext is a façade for HBase operations
+  * like bulk put, get, increment, delete, and scan
+  *
+  * HBaseContext will take the responsibilities
+  * of disseminating the configuration information
+  * to the working and managing the life cycle of HConnections.
+ */
+class HBaseContext(@transient private val sc: SparkContext,
+                   @transient private val config: Configuration,
+                   val tmpHdfsConfgFile: String = null)
+  extends Serializable {
+
+  @transient var credentials = UserGroupInformation.getCurrentUser().getCredentials
+  @transient var tmpHdfsConfiguration:Configuration = config
+  @transient var appliedCredentials = false
+  @transient val job = Job.getInstance(config)
+  TableMapReduceUtil.initCredentials(job)
+  val broadcastedConf = sc.broadcast(new SerializableWritable(config))
+  val credentialsConf = sc.broadcast(new SerializableWritable(job.getCredentials))
+
+  LatestHBaseContextCache.latest = this
+
+  if (tmpHdfsConfgFile != null && config != null) {
+    val fs = FileSystem.newInstance(config)
+    val tmpPath = new Path(tmpHdfsConfgFile)
+    if (!fs.exists(tmpPath)) {
+      val outputStream = fs.create(tmpPath)
+      config.write(outputStream)
+      outputStream.close()
+    } else {
+//      logWarning("tmpHdfsConfigDir " + tmpHdfsConfgFile + " exist!!")
+    }
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark RDD foreachPartition.
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * @param rdd  Original RDD with data to iterate over
+   * @param f    Function to be given a iterator to iterate through
+   *             the RDD values and a HConnection object to interact
+   *             with HBase
+   */
+  def foreachPartition[T](rdd: RDD[T],
+                          f: (Iterator[T], Connection) => Unit):Unit = {
+    rdd.foreachPartition(
+      it => hbaseForeachPartition(broadcastedConf, it, f))
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark Streaming dStream foreach
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * @param dstream  Original DStream with data to iterate over
+   * @param f        Function to be given a iterator to iterate through
+   *                 the DStream values and a HConnection object to
+   *                 interact with HBase
+   */
+  def foreachPartition[T](dstream: DStream[T],
+                    f: (Iterator[T], Connection) => Unit):Unit = {
+    dstream.foreachRDD((rdd, time) => {
+      foreachPartition(rdd, f)
+    })
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark RDD mapPartition.
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * @param rdd  Original RDD with data to iterate over
+   * @param mp   Function to be given a iterator to iterate through
+   *             the RDD values and a HConnection object to interact
+   *             with HBase
+   * @return     Returns a new RDD generated by the user definition
+   *             function just like normal mapPartition
+   */
+  def mapPartitions[T, R: ClassTag](rdd: RDD[T],
+                                   mp: (Iterator[T], Connection) => Iterator[R]): RDD[R] = {
+
+    rdd.mapPartitions[R](it => hbaseMapPartition[T, R](broadcastedConf,
+      it,
+      mp))
+
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark Streaming DStream
+   * foreachPartition.
+   *
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * Note: Make sure to partition correctly to avoid memory issue when
+   *       getting data from HBase
+   *
+   * @param dstream  Original DStream with data to iterate over
+   * @param f       Function to be given a iterator to iterate through
+   *                 the DStream values and a HConnection object to
+   *                 interact with HBase
+   * @return         Returns a new DStream generated by the user
+   *                 definition function just like normal mapPartition
+   */
+  def streamForeachPartition[T](dstream: DStream[T],
+                                f: (Iterator[T], Connection) => Unit): Unit = {
+
+    dstream.foreachRDD(rdd => this.foreachPartition(rdd, f))
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark Streaming DStream
+   * mapPartition.
+   *
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * Note: Make sure to partition correctly to avoid memory issue when
+   *       getting data from HBase
+   *
+   * @param dstream  Original DStream with data to iterate over
+   * @param f       Function to be given a iterator to iterate through
+   *                 the DStream values and a HConnection object to
+   *                 interact with HBase
+   * @return         Returns a new DStream generated by the user
+   *                 definition function just like normal mapPartition
+   */
+  def streamMapPartitions[T, U: ClassTag](dstream: DStream[T],
+                                f: (Iterator[T], Connection) => Iterator[U]):
+  DStream[U] = {
+    dstream.mapPartitions(it => hbaseMapPartition[T, U](
+      broadcastedConf,
+      it,
+      f))
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.foreachPartition method.
+   *
+   * It allow addition support for a user to take RDD
+   * and generate puts and send them to HBase.
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param rdd       Original RDD with data to iterate over
+   * @param tableName The name of the table to put into
+   * @param f         Function to convert a value in the RDD to a HBase Put
+   */
+  def bulkPut[T](rdd: RDD[T], tableName: TableName, f: (T) => Put) {
+
+    val tName = tableName.getName
+    rdd.foreachPartition(
+      it => hbaseForeachPartition[T](
+        broadcastedConf,
+        it,
+        (iterator, connection) => {
+          val m = connection.getBufferedMutator(TableName.valueOf(tName))
+          iterator.foreach(T => m.mutate(f(T)))
+          m.flush()
+          m.close()
+        }))
+  }
+
+  def applyCreds[T] (configBroadcast: Broadcast[SerializableWritable[Configuration]]){
+    credentials = UserGroupInformation.getCurrentUser().getCredentials
+
+//    logDebug("appliedCredentials:" + appliedCredentials + ",credentials:" + credentials)
+
+    if (!appliedCredentials && credentials != null) {
+      appliedCredentials = true
+
+      @transient val ugi = UserGroupInformation.getCurrentUser
+      ugi.addCredentials(credentials)
+      // specify that this is a proxy user
+      ugi.setAuthenticationMethod(AuthenticationMethod.PROXY)
+
+      ugi.addCredentials(credentialsConf.value.value)
+    }
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamMapPartition method.
+   *
+   * It allow addition support for a user to take a DStream and
+   * generate puts and send them to HBase.
+   *
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param dstream    Original DStream with data to iterate over
+   * @param tableName  The name of the table to put into
+   * @param f          Function to convert a value in
+   *                   the DStream to a HBase Put
+   */
+  def streamBulkPut[T](dstream: DStream[T],
+                       tableName: TableName,
+                       f: (T) => Put) = {
+    val tName = tableName.getName
+    dstream.foreachRDD((rdd, time) => {
+      bulkPut(rdd, TableName.valueOf(tName), f)
+    })
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.foreachPartition method.
+   *
+   * It allow addition support for a user to take a RDD and generate delete
+   * and send them to HBase.  The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param rdd       Original RDD with data to iterate over
+   * @param tableName The name of the table to delete from
+   * @param f         Function to convert a value in the RDD to a
+   *                  HBase Deletes
+   * @param batchSize       The number of delete to batch before sending to HBase
+   */
+  def bulkDelete[T](rdd: RDD[T], tableName: TableName,
+                    f: (T) => Delete, batchSize: Integer) {
+    bulkMutation(rdd, tableName, f, batchSize)
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamBulkMutation method.
+   *
+   * It allow addition support for a user to take a DStream and
+   * generate Delete and send them to HBase.
+   *
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param dstream    Original DStream with data to iterate over
+   * @param tableName  The name of the table to delete from
+   * @param f          function to convert a value in the DStream to a
+   *                   HBase Delete
+   * @param batchSize        The number of deletes to batch before sending to HBase
+   */
+  def streamBulkDelete[T](dstream: DStream[T],
+                          tableName: TableName,
+                          f: (T) => Delete,
+                          batchSize: Integer) = {
+    streamBulkMutation(dstream, tableName, f, batchSize)
+  }
+
+  /**
+   *  Under lining function to support all bulk mutations
+   *
+   *  May be opened up if requested
+   */
+  private def bulkMutation[T](rdd: RDD[T], tableName: TableName,
+                              f: (T) => Mutation, batchSize: Integer) {
+
+    val tName = tableName.getName
+    rdd.foreachPartition(
+      it => hbaseForeachPartition[T](
+        broadcastedConf,
+        it,
+        (iterator, connection) => {
+          val table = connection.getTable(TableName.valueOf(tName))
+          val mutationList = new java.util.ArrayList[Mutation]
+          iterator.foreach(T => {
+            mutationList.add(f(T))
+            if (mutationList.size >= batchSize) {
+              table.batch(mutationList, null)
+              mutationList.clear()
+            }
+          })
+          if (mutationList.size() > 0) {
+            table.batch(mutationList, null)
+            mutationList.clear()
+          }
+          table.close()
+        }))
+  }
+
+  /**
+   *  Under lining function to support all bulk streaming mutations
+   *
+   *  May be opened up if requested
+   */
+  private def streamBulkMutation[T](dstream: DStream[T],
+                                    tableName: TableName,
+                                    f: (T) => Mutation,
+                                    batchSize: Integer) = {
+    val tName = tableName.getName
+    dstream.foreachRDD((rdd, time) => {
+      bulkMutation(rdd, TableName.valueOf(tName), f, batchSize)
+    })
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.mapPartition method.
+   *
+   * It allow addition support for a user to take a RDD and generates a
+   * new RDD based on Gets and the results they bring back from HBase
+   *
+   * @param rdd     Original RDD with data to iterate over
+   * @param tableName        The name of the table to get from
+   * @param makeGet    function to convert a value in the RDD to a
+   *                   HBase Get
+   * @param convertResult This will convert the HBase Result object to
+   *                   what ever the user wants to put in the resulting
+   *                   RDD
+   * return            new RDD that is created by the Get to HBase
+   */
+  def bulkGet[T, U: ClassTag](tableName: TableName,
+                    batchSize: Integer,
+                    rdd: RDD[T],
+                    makeGet: (T) => Get,
+                    convertResult: (Result) => U): RDD[U] = {
+
+    val getMapPartition = new GetMapPartition(tableName,
+      batchSize,
+      makeGet,
+      convertResult)
+
+    rdd.mapPartitions[U](it =>
+      hbaseMapPartition[T, U](
+        broadcastedConf,
+        it,
+        getMapPartition.run))
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamMap method.
+   *
+   * It allow addition support for a user to take a DStream and
+   * generates a new DStream based on Gets and the results
+   * they bring back from HBase
+   *
+   * @param tableName     The name of the table to get from
+   * @param batchSize     The number of Gets to be sent in a single batch
+   * @param dStream       Original DStream with data to iterate over
+   * @param makeGet       Function to convert a value in the DStream to a
+   *                      HBase Get
+   * @param convertResult This will convert the HBase Result object to
+   *                      what ever the user wants to put in the resulting
+   *                      DStream
+   * @return              A new DStream that is created by the Get to HBase
+   */
+  def streamBulkGet[T, U: ClassTag](tableName: TableName,
+                                    batchSize: Integer,
+                                    dStream: DStream[T],
+                                    makeGet: (T) => Get,
+                                    convertResult: (Result) => U): DStream[U] = {
+
+    val getMapPartition = new GetMapPartition(tableName,
+      batchSize,
+      makeGet,
+      convertResult)
+
+    dStream.mapPartitions[U](it => hbaseMapPartition[T, U](
+      broadcastedConf,
+      it,
+      getMapPartition.run))
+  }
+
+  /**
+   * This function will use the native HBase TableInputFormat with the
+   * given scan object to generate a new RDD
+   *
+   *  @param tableName the name of the table to scan
+   *  @param scan      the HBase scan object to use to read data from HBase
+   *  @param f         function to convert a Result object from HBase into
+   *                   what the user wants in the final generated RDD
+   *  @return          new RDD with results from scan
+   */
+  def hbaseRDD[U: ClassTag](tableName: TableName, scan: Scan,
+                            f: ((ImmutableBytesWritable, Result)) => U): RDD[U] = {
+
+    val job: Job = Job.getInstance(getConf(broadcastedConf))
+
+    TableMapReduceUtil.initCredentials(job)
+    TableMapReduceUtil.initTableMapperJob(tableName, scan,
+      classOf[IdentityTableMapper], null, null, job)
+
+    sc.newAPIHadoopRDD(job.getConfiguration,
+      classOf[TableInputFormat],
+      classOf[ImmutableBytesWritable],
+      classOf[Result]).map(f)
+  }
+
+  /**
+   * A overloaded version of HBaseContext hbaseRDD that defines the
+   * type of the resulting RDD
+   *
+   *  @param tableName the name of the table to scan
+   *  @param scans     the HBase scan object to use to read data from HBase
+   *  @return          New RDD with results from scan
+   *
+   */
+  def hbaseRDD(tableName: TableName, scans: Scan):
+  RDD[(ImmutableBytesWritable, Result)] = {
+
+    hbaseRDD[(ImmutableBytesWritable, Result)](
+      tableName,
+      scans,
+      (r: (ImmutableBytesWritable, Result)) => r)
+  }
+
+  /**
+   *  underlining wrapper all foreach functions in HBaseContext
+   */
+  private def hbaseForeachPartition[T](configBroadcast:
+                                       Broadcast[SerializableWritable[Configuration]],
+                                        it: Iterator[T],
+                                        f: (Iterator[T], Connection) => Unit) = {
+
+    val config = getConf(configBroadcast)
+
+    applyCreds(configBroadcast)
+    // specify that this is a proxy user
+    val connection = ConnectionFactory.createConnection(config)
+    f(it, connection)
+    connection.close()
+  }
+
+  private def getConf(configBroadcast: Broadcast[SerializableWritable[Configuration]]):
+  Configuration = {
+
+    if (tmpHdfsConfiguration == null && tmpHdfsConfgFile != null) {
+      val fs = FileSystem.newInstance(SparkHadoopUtil.get.conf)
+      val inputStream = fs.open(new Path(tmpHdfsConfgFile))
+      tmpHdfsConfiguration = new Configuration(false)
+      tmpHdfsConfiguration.readFields(inputStream)
+      inputStream.close()
+    }
+
+    if (tmpHdfsConfiguration == null) {
+      try {
+        tmpHdfsConfiguration = configBroadcast.value.value
+      } catch {
+        case ex: Exception =>
+//          logError("Unable to getConfig from broadcast", ex)
+      }
+    }
+    tmpHdfsConfiguration
+  }
+
+  /**
+   *  underlining wrapper all mapPartition functions in HBaseContext
+   *
+   */
+  private def hbaseMapPartition[K, U](
+                                       configBroadcast:
+                                       Broadcast[SerializableWritable[Configuration]],
+                                       it: Iterator[K],
+                                       mp: (Iterator[K], Connection) =>
+                                         Iterator[U]): Iterator[U] = {
+
+    val config = getConf(configBroadcast)
+    applyCreds(configBroadcast)
+
+    val connection = ConnectionFactory.createConnection(config)
+    val res = mp(it, connection)
+    connection.close()
+    res
+
+  }
+
+  /**
+   *  underlining wrapper all get mapPartition functions in HBaseContext
+   */
+  private class GetMapPartition[T, U](tableName: TableName,
+                                      batchSize: Integer,
+                                      makeGet: (T) => Get,
+                                      convertResult: (Result) => U)
+    extends Serializable {
+
+    val tName = tableName.getName
+
+    def run(iterator: Iterator[T], connection: Connection): Iterator[U] = {
+      val table = connection.getTable(TableName.valueOf(tName))
+
+      val gets = new java.util.ArrayList[Get]()
+      var res = List[U]()
+
+      while (iterator.hasNext) {
+        gets.add(makeGet(iterator.next()))
+
+        if (gets.size() == batchSize) {
+          val results = table.get(gets)
+          res = res ++ results.map(convertResult)
+          gets.clear()
+        }
+      }
+      if (gets.size() > 0) {
+        val results = table.get(gets)
+        res = res ++ results.map(convertResult)
+        gets.clear()
+      }
+      table.close()
+      res.iterator
+    }
+  }
+
+  /**
+   * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+   *
+   * This method is used to keep ClassTags out of the external Java API, as
+   * the Java compiler cannot produce them automatically. While this
+   * ClassTag-faking does please the compiler, it can cause problems at runtime
+   * if the Scala API relies on ClassTags for correctness.
+   *
+   * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
+   * just worse performance or security issues.
+   * For instance, an Array of AnyRef can hold any type T, but may lose primitive
+   * specialization.
+   */
+  private[spark]
+  def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
+
+  /**
+   * A Spark Implementation of HBase Bulk load
+   *
+   * This will take the content from an existing RDD then sort and shuffle
+   * it with respect to region splits.  The result of that sort and shuffle
+   * will be written to HFiles.
+   *
+   * After this function is executed the user will have to call
+   * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
+   *
+   * Also note this version of bulk load is different from past versions in
+   * that it includes the qualifier as part of the sort process. The
+   * reason for this is to be able to support rows will very large number
+   * of columns.
+   *
+   * @param rdd                            The RDD we are bulk loading from
+   * @param tableName                      The HBase table we are loading into
+   * @param startKeys
+   * @param flatMap                        A flapMap function that will make every
+   *                                       row in the RDD
+   *                                       into N cells for the bulk load
+   * @param stagingDir                     The location on the FileSystem to bulk load into
+   * @param familyHFileWriteOptionsMap     Options that will define how the HFile for a
+   *                                       column family is written
+   * @param compactionExclude              Compaction excluded for the HFiles
+   * @param maxSize                        Max size for the HFiles before they roll
+   * @tparam T                             The Type of values in the original RDD
+   */
+  def bulkLoad[T](rdd:RDD[T],
+                  tableName: TableName,
+                  startKeys: Array[Array[Byte]],
+                  flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
+                  stagingDir:String,
+                  familyHFileWriteOptionsMap: util.Map[Array[Byte], FamilyHFileWriteOptions] = new util.HashMap[Array[Byte], FamilyHFileWriteOptions],
+                  compactionExclude: Boolean = false,
+                  maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):
+  Unit = {
+//    val conn = ConnectionFactory.createConnection(config)
+//    val regionLocator = conn.getRegionLocator(tableName)
+//    val startKeys = regionLocator.getStartKeys
+    val defaultCompressionStr = config.get("hfile.compression",
+      Compression.Algorithm.NONE.getName)
+    val defaultCompression = Compression.getCompressionAlgorithmByName(defaultCompressionStr)
+//      HFileWriterImpl
+//      .compressionByName(defaultCompressionStr)
+    val now = System.currentTimeMillis()
+    val tableNameByteArray = tableName.getName
+
+    val familyHFileWriteOptionsMapInternal =
+      new util.HashMap[ByteArrayWrapper, FamilyHFileWriteOptions]
+
+    val entrySetIt = familyHFileWriteOptionsMap.entrySet().iterator()
+
+    while (entrySetIt.hasNext) {
+      val entry = entrySetIt.next()
+      familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(entry.getKey), entry.getValue)
+    }
+
+    /*
+     *  This will return a new HFile writer when requested
+     *
+     * @param family       column family
+     * @param conf         configuration to connect to HBase
+     * @param favoredNodes nodes that we would like to write too
+     * @param fs           FileSystem object where we will be writing the HFiles to
+     * @return WriterLength object
+     */
+    def getNewWriter(family: Array[Byte], conf: Configuration,
+                     favoredNodes: Array[InetSocketAddress],
+                     fs:FileSystem,
+                     familydir:Path): WriterLength = {
+
+
+      var familyOptions = familyHFileWriteOptionsMapInternal.get(new ByteArrayWrapper(family))
+
+      if (familyOptions == null) {
+        familyOptions = new FamilyHFileWriteOptions(defaultCompression.toString,
+          BloomType.NONE.toString, HConstants.DEFAULT_BLOCKSIZE, DataBlockEncoding.NONE.toString)
+        familyHFileWriteOptionsMapInternal.put(new ByteArrayWrapper(family), familyOptions)
+      }
+
+      val tempConf = new Configuration(conf)
+      tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f)
+      val contextBuilder = new HFileContextBuilder()
+        .withCompression(Algorithm.valueOf(familyOptions.compression))
+        .withChecksumType(HStore.getChecksumType(conf))
+        .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))
+        .withBlockSize(familyOptions.blockSize)
+      contextBuilder.withDataBlockEncoding(DataBlockEncoding.
+        valueOf(familyOptions.dataBlockEncoding))
+      val hFileContext = contextBuilder.build()
+
+      if (null == favoredNodes) {
+        new WriterLength(0, new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs)
+          .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
+          .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext).build())
+//          .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext).build())
+      } else {
+        new WriterLength(0,
+          new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), new HFileSystem(fs))
+          .withOutputDir(familydir).withBloomType(BloomType.valueOf(familyOptions.bloomType))
+            .withComparator(KeyValue.COMPARATOR).withFileContext(hFileContext)
+//          .withComparator(CellComparator.COMPARATOR).withFileContext(hFileContext)
+          .withFavoredNodes(favoredNodes).build())
+      }
+    }
+
+    val regionSplitPartitioner =
+      new BulkLoadPartitioner(startKeys)
+
+    //This is where all the magic happens
+    //Here we are going to do the following things
+    // 1. FlapMap every row in the RDD into key column value tuples
+    // 2. Then we are going to repartition sort and shuffle
+    // 3. Finally we are going to write out our HFiles
+    rdd.flatMap( r => flatMap(r)).
+      repartitionAndSortWithinPartitions(regionSplitPartitioner).
+      hbaseForeachPartition(this, (it, conn) => {
+
+      val conf = broadcastedConf.value.value
+      val fs = FileSystem.get(conf)
+      val writerMap = new mutable.HashMap[ByteArrayWrapper, WriterLength]
+      var previousRow:Array[Byte] = HConstants.EMPTY_BYTE_ARRAY
+      var rollOverRequested = false
+
+      /*
+       * This will roll all writers
+       */
+      def rollWriters(): Unit = {
+        writerMap.values.foreach( wl => {
+          if (wl.writer != null) {
+//            logDebug("Writer=" + wl.writer.getPath +
+//              (if (wl.written == 0) "" else ", wrote=" + wl.written))
+            close(wl.writer)
+          }
+        })
+        writerMap.clear()
+        rollOverRequested = false
+      }
+
+      /*
+       * This function will close a given HFile writer
+       * @param w The writer to close
+       */
+      def close(w:StoreFile.Writer): Unit = {
+        if (w != null) {
+          w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
+            Bytes.toBytes(System.currentTimeMillis()))
+          w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
+            Bytes.toBytes(regionSplitPartitioner.getPartition(previousRow)))
+          w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY,
+            Bytes.toBytes(true))
+          w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY,
+            Bytes.toBytes(compactionExclude))
+          w.appendTrackedTimestampsToMetadata()
+          w.close()
+        }
+      }
+
+      //Here is where we finally iterate through the data in this partition of the
+      //RDD that has been sorted and partitioned
+      it.foreach{ case (keyFamilyQualifier, cellValue:Array[Byte]) =>
+
+        //This will get a writer for the column family
+        //If there is no writer for a given column family then
+        //it will get created here.
+        val wl = writerMap.getOrElseUpdate(new ByteArrayWrapper(keyFamilyQualifier.family), {
+
+          val familyDir = new Path(stagingDir, Bytes.toString(keyFamilyQualifier.family))
+
+          fs.mkdirs(familyDir)
+
+          val loc:HRegionLocation = {
+            try {
+              val locator =
+                conn.getRegionLocator(TableName.valueOf(tableNameByteArray))
+              locator.getRegionLocation(keyFamilyQualifier.rowKey)
+            } catch {
+              case e: Throwable =>
+//              logWarning("there's something wrong when locating rowkey: " +
+//                Bytes.toString(keyFamilyQualifier.rowKey))
+                null
+            }
+          }
+          if (null == loc) {
+//            if (log.isTraceEnabled) {
+//              logTrace("failed to get region location, so use default writer: " +
+//                Bytes.toString(keyFamilyQualifier.rowKey))
+//            }
+            getNewWriter(family = keyFamilyQualifier.family, conf = conf, favoredNodes = null,
+              fs = fs, familydir = familyDir)
+          } else {
+//            if (log.isDebugEnabled) {
+//              logDebug("first rowkey: [" + Bytes.toString(keyFamilyQualifier.rowKey) + "]")
+//            }
+            val initialIsa =
+              new InetSocketAddress(loc.getHostname, loc.getPort)
+            if (initialIsa.isUnresolved) {
+//              if (log.isTraceEnabled) {
+//                logTrace("failed to resolve bind address: " + loc.getHostname + ":"
+//                  + loc.getPort + ", so use default writer")
+//              }
+              getNewWriter(keyFamilyQualifier.family, conf, null, fs, familyDir)
+            } else {
+//              if(log.isDebugEnabled) {
+//                logDebug("use favored nodes writer: " + initialIsa.getHostString)
+//              }
+              getNewWriter(keyFamilyQualifier.family, conf,
+                Array[InetSocketAddress](initialIsa), fs, familyDir)
+            }
+          }
+        })
+
+        val keyValue =new KeyValue(keyFamilyQualifier.rowKey,
+          keyFamilyQualifier.family,
+          keyFamilyQualifier.qualifier,
+          now,cellValue)
+
+        wl.writer.append(keyValue)
+        wl.written += keyValue.getLength
+
+        rollOverRequested = rollOverRequested || wl.written > maxSize
+
+        //This will only roll if we have at least one column family file that is
+        //bigger then maxSize and we have finished a given row key
+        if (rollOverRequested && Bytes.compareTo(previousRow, keyFamilyQualifier.rowKey) != 0) {
+          rollWriters()
+        }
+
+        previousRow = keyFamilyQualifier.rowKey
+      }
+      //We have finished all the data so lets close up the writers
+      rollWriters()
+    })
+  }
+
+  /**
+   * This is a wrapper class around StoreFile.Writer.  The reason for the
+   * wrapper is to keep the length of the file along side the writer
+   *
+   * @param written The writer to be wrapped
+   * @param writer  The number of bytes written to the writer
+   */
+  class WriterLength(var written:Long, val writer:StoreFile.Writer)
+
+  /**
+   * This is a wrapper over a byte array so it can work as
+   * a key in a hashMap
+   *
+   * @param o1 The Byte Array value
+   */
+  class ByteArrayWrapper (val o1:Array[Byte])
+    extends Comparable[ByteArrayWrapper] with Serializable {
+    override def compareTo(o2: ByteArrayWrapper): Int = {
+      Bytes.compareTo(o1,o2.o1)
+    }
+    override def equals(o2: Any): Boolean = {
+      o2 match {
+        case wrapper: ByteArrayWrapper =>
+          Bytes.equals(o1, wrapper.o1)
+        case _ =>
+          false
+      }
+    }
+    override def hashCode():Int = {
+      Bytes.hashCode(o1)
+    }
+  }
+}
+
+object LatestHBaseContextCache {
+  var latest:HBaseContext = null
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseDStreamFunctions.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseDStreamFunctions.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseDStreamFunctions.scala
new file mode 100644
index 0000000..9faaecd
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseDStreamFunctions.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.s2graph.s2jobs.spark
+
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.spark.streaming.dstream.DStream
+
+import scala.reflect.ClassTag
+
+/**
+ * HBaseDStreamFunctions contains a set of implicit functions that can be
+ * applied to a Spark DStream so that we can easily interact with HBase
+ */
+object HBaseDStreamFunctions {
+
+  /**
+   * These are implicit methods for a DStream that contains any type of
+   * data.
+   *
+   * @param dStream  This is for dStreams of any type
+   * @tparam T       Type T
+   */
+  implicit class GenericHBaseDStreamFunctions[T](val dStream: DStream[T]) {
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * put.  This will not return a new Stream.  Think of it like a foreach
+     *
+     * @param hc         The hbaseContext object to identify which
+     *                   HBase cluster connection to use
+     * @param tableName  The tableName that the put will be sent to
+     * @param f          The function that will turn the DStream values
+     *                   into HBase Put objects.
+     */
+    def hbaseBulkPut(hc: HBaseContext,
+                     tableName: TableName,
+                     f: (T) => Put): Unit = {
+      hc.streamBulkPut(dStream, tableName, f)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * get.  This will return a new DStream.  Think about it as a DStream map
+     * function.  In that every DStream value will get a new value out of
+     * HBase.  That new value will populate the newly generated DStream.
+     *
+     * @param hc             The hbaseContext object to identify which
+     *                       HBase cluster connection to use
+     * @param tableName      The tableName that the put will be sent to
+     * @param batchSize      How many gets to execute in a single batch
+     * @param f              The function that will turn the RDD values
+     *                       in HBase Get objects
+     * @param convertResult  The function that will convert a HBase
+     *                       Result object into a value that will go
+     *                       into the resulting DStream
+     * @tparam R             The type of Object that will be coming
+     *                       out of the resulting DStream
+     * @return               A resulting DStream with type R objects
+     */
+    def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
+                     tableName: TableName,
+                     batchSize:Int, f: (T) => Get, convertResult: (Result) => R):
+    DStream[R] = {
+      hc.streamBulkGet[T, R](tableName, batchSize, dStream, f, convertResult)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * get.  This will return a new DStream.  Think about it as a DStream map
+     * function.  In that every DStream value will get a new value out of
+     * HBase.  That new value will populate the newly generated DStream.
+     *
+     * @param hc             The hbaseContext object to identify which
+     *                       HBase cluster connection to use
+     * @param tableName      The tableName that the put will be sent to
+     * @param batchSize      How many gets to execute in a single batch
+     * @param f              The function that will turn the RDD values
+     *                       in HBase Get objects
+     * @return               A resulting DStream with type R objects
+     */
+    def hbaseBulkGet(hc: HBaseContext,
+                     tableName: TableName, batchSize:Int,
+                     f: (T) => Get): DStream[(ImmutableBytesWritable, Result)] = {
+        hc.streamBulkGet[T, (ImmutableBytesWritable, Result)](
+          tableName, batchSize, dStream, f,
+          result => (new ImmutableBytesWritable(result.getRow), result))
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * Delete.  This will not return a new DStream.
+     *
+     * @param hc         The hbaseContext object to identify which HBase
+     *                   cluster connection to use
+     * @param tableName  The tableName that the deletes will be sent to
+     * @param f          The function that will convert the DStream value into
+     *                   a HBase Delete Object
+     * @param batchSize  The number of Deletes to be sent in a single batch
+     */
+    def hbaseBulkDelete(hc: HBaseContext,
+                        tableName: TableName,
+                        f:(T) => Delete, batchSize:Int): Unit = {
+      hc.streamBulkDelete(dStream, tableName, f, batchSize)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's
+     * foreachPartition method.  This will ack very much like a normal DStream
+     * foreach method but for the fact that you will now have a HBase connection
+     * while iterating through the values.
+     *
+     * @param hc  The hbaseContext object to identify which HBase
+     *            cluster connection to use
+     * @param f   This function will get an iterator for a Partition of an
+     *            DStream along with a connection object to HBase
+     */
+    def hbaseForeachPartition(hc: HBaseContext,
+                              f: (Iterator[T], Connection) => Unit): Unit = {
+      hc.streamForeachPartition(dStream, f)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's
+     * mapPartitions method.  This will ask very much like a normal DStream
+     * map partitions method but for the fact that you will now have a
+     * HBase connection while iterating through the values
+     *
+     * @param hc  The hbaseContext object to identify which HBase
+     *            cluster connection to use
+     * @param f   This function will get an iterator for a Partition of an
+     *            DStream along with a connection object to HBase
+     * @tparam R  This is the type of objects that will go into the resulting
+     *            DStream
+     * @return    A resulting DStream of type R
+     */
+    def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
+                                        f: (Iterator[T], Connection) => Iterator[R]):
+    DStream[R] = {
+      hc.streamMapPartitions(dStream, f)
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseRDDFunctions.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseRDDFunctions.scala
new file mode 100644
index 0000000..328587c
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/HBaseRDDFunctions.scala
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import java.util
+
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.{HConstants, TableName}
+import org.apache.spark.rdd.RDD
+
+import scala.reflect.ClassTag
+
+/**
+ * HBaseRDDFunctions contains a set of implicit functions that can be
+ * applied to a Spark RDD so that we can easily interact with HBase
+ */
+object HBaseRDDFunctions
+{
+
+  /**
+   * These are implicit methods for a RDD that contains any type of
+   * data.
+   *
+   * @param rdd This is for rdd of any type
+   * @tparam T  This is any type
+   */
+  implicit class GenericHBaseRDDFunctions[T](val rdd: RDD[T]) {
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * put.  This will not return a new RDD.  Think of it like a foreach
+     *
+     * @param hc         The hbaseContext object to identify which
+     *                   HBase cluster connection to use
+     * @param tableName  The tableName that the put will be sent to
+     * @param f          The function that will turn the RDD values
+     *                   into HBase Put objects.
+     */
+    def hbaseBulkPut(hc: HBaseContext,
+                     tableName: TableName,
+                     f: (T) => Put): Unit = {
+      hc.bulkPut(rdd, tableName, f)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * get.  This will return a new RDD.  Think about it as a RDD map
+     * function.  In that every RDD value will get a new value out of
+     * HBase.  That new value will populate the newly generated RDD.
+     *
+     * @param hc             The hbaseContext object to identify which
+     *                       HBase cluster connection to use
+     * @param tableName      The tableName that the put will be sent to
+     * @param batchSize      How many gets to execute in a single batch
+     * @param f              The function that will turn the RDD values
+     *                       in HBase Get objects
+     * @param convertResult  The function that will convert a HBase
+     *                       Result object into a value that will go
+     *                       into the resulting RDD
+     * @tparam R             The type of Object that will be coming
+     *                       out of the resulting RDD
+     * @return               A resulting RDD with type R objects
+     */
+    def hbaseBulkGet[R: ClassTag](hc: HBaseContext,
+                            tableName: TableName, batchSize:Int,
+                            f: (T) => Get, convertResult: (Result) => R): RDD[R] = {
+      hc.bulkGet[T, R](tableName, batchSize, rdd, f, convertResult)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * get.  This will return a new RDD.  Think about it as a RDD map
+     * function.  In that every RDD value will get a new value out of
+     * HBase.  That new value will populate the newly generated RDD.
+     *
+     * @param hc             The hbaseContext object to identify which
+     *                       HBase cluster connection to use
+     * @param tableName      The tableName that the put will be sent to
+     * @param batchSize      How many gets to execute in a single batch
+     * @param f              The function that will turn the RDD values
+     *                       in HBase Get objects
+     * @return               A resulting RDD with type R objects
+     */
+    def hbaseBulkGet(hc: HBaseContext,
+                                  tableName: TableName, batchSize:Int,
+                                  f: (T) => Get): RDD[(ImmutableBytesWritable, Result)] = {
+      hc.bulkGet[T, (ImmutableBytesWritable, Result)](tableName,
+        batchSize, rdd, f,
+        result => if (result != null && result.getRow != null) {
+          (new ImmutableBytesWritable(result.getRow), result)
+        } else {
+          null
+        })
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's bulk
+     * Delete.  This will not return a new RDD.
+     *
+     * @param hc         The hbaseContext object to identify which HBase
+     *                   cluster connection to use
+     * @param tableName  The tableName that the deletes will be sent to
+     * @param f          The function that will convert the RDD value into
+     *                   a HBase Delete Object
+     * @param batchSize  The number of Deletes to be sent in a single batch
+     */
+    def hbaseBulkDelete(hc: HBaseContext,
+                        tableName: TableName, f:(T) => Delete, batchSize:Int): Unit = {
+      hc.bulkDelete(rdd, tableName, f, batchSize)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's
+     * foreachPartition method.  This will ack very much like a normal RDD
+     * foreach method but for the fact that you will now have a HBase connection
+     * while iterating through the values.
+     *
+     * @param hc  The hbaseContext object to identify which HBase
+     *            cluster connection to use
+     * @param f   This function will get an iterator for a Partition of an
+     *            RDD along with a connection object to HBase
+     */
+    def hbaseForeachPartition(hc: HBaseContext,
+                              f: (Iterator[T], Connection) => Unit): Unit = {
+      hc.foreachPartition(rdd, f)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's
+     * mapPartitions method.  This will ask very much like a normal RDD
+     * map partitions method but for the fact that you will now have a
+     * HBase connection while iterating through the values
+     *
+     * @param hc  The hbaseContext object to identify which HBase
+     *            cluster connection to use
+     * @param f   This function will get an iterator for a Partition of an
+     *            RDD along with a connection object to HBase
+     * @tparam R  This is the type of objects that will go into the resulting
+     *            RDD
+     * @return    A resulting RDD of type R
+     */
+    def hbaseMapPartitions[R: ClassTag](hc: HBaseContext,
+                                        f: (Iterator[T], Connection) => Iterator[R]):
+    RDD[R] = {
+      hc.mapPartitions[T,R](rdd, f)
+    }
+
+    /**
+     * Implicit method that gives easy access to HBaseContext's
+     * bulkLoad method.
+     *
+     * A Spark Implementation of HBase Bulk load
+     *
+     * This will take the content from an existing RDD then sort and shuffle
+     * it with respect to region splits.  The result of that sort and shuffle
+     * will be written to HFiles.
+     *
+     * After this function is executed the user will have to call
+     * LoadIncrementalHFiles.doBulkLoad(...) to move the files into HBase
+     *
+     * Also note this version of bulk load is different from past versions in
+     * that it includes the qualifier as part of the sort process. The
+     * reason for this is to be able to support rows will very large number
+     * of columns.
+     *
+     * @param tableName                      The HBase table we are loading into
+     * @param flatMap                        A flapMap function that will make every row in the RDD
+     *                                       into N cells for the bulk load
+     * @param stagingDir                     The location on the FileSystem to bulk load into
+     * @param familyHFileWriteOptionsMap     Options that will define how the HFile for a
+     *                                       column family is written
+     * @param compactionExclude              Compaction excluded for the HFiles
+     * @param maxSize                        Max size for the HFiles before they roll
+     */
+    def hbaseBulkLoad(hc: HBaseContext,
+                         tableName: TableName,
+                         startKeys: Array[Array[Byte]],
+                         flatMap: (T) => Iterator[(KeyFamilyQualifier, Array[Byte])],
+                         stagingDir:String,
+                         familyHFileWriteOptionsMap:
+                         util.Map[Array[Byte], FamilyHFileWriteOptions] =
+                         new util.HashMap[Array[Byte], FamilyHFileWriteOptions](),
+                         compactionExclude: Boolean = false,
+                         maxSize:Long = HConstants.DEFAULT_MAX_FILE_SIZE):Unit = {
+      hc.bulkLoad(rdd, tableName, startKeys,
+        flatMap, stagingDir, familyHFileWriteOptionsMap,
+        compactionExclude, maxSize)
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/JavaHBaseContext.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/JavaHBaseContext.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/JavaHBaseContext.scala
new file mode 100644
index 0000000..d2dc7d3
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/JavaHBaseContext.scala
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.TableName
+import org.apache.hadoop.hbase.client._
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.spark.api.java.function.{FlatMapFunction, Function, VoidFunction}
+import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
+import org.apache.spark.streaming.api.java.JavaDStream
+
+import scala.collection.JavaConversions._
+import scala.reflect.ClassTag
+
+/**
+ * This is the Java Wrapper over HBaseContext which is written in
+ * Scala.  This class will be used by developers that want to
+ * work with Spark or Spark Streaming in Java
+ *
+ * @param jsc    This is the JavaSparkContext that we will wrap
+ * @param config This is the config information to out HBase cluster
+ */
+class JavaHBaseContext(@transient private val jsc: JavaSparkContext,
+                       @transient private val config: Configuration) extends Serializable {
+  val hbaseContext = new HBaseContext(jsc.sc, config)
+
+  /**
+   * A simple enrichment of the traditional Spark javaRdd foreachPartition.
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * @param javaRdd Original javaRdd with data to iterate over
+   * @param f       Function to be given a iterator to iterate through
+   *                the RDD values and a HConnection object to interact
+   *                with HBase
+   */
+  def foreachPartition[T](javaRdd: JavaRDD[T],
+                          f: VoidFunction[(java.util.Iterator[T], Connection)]) = {
+
+    hbaseContext.foreachPartition(javaRdd.rdd,
+      (it: Iterator[T], conn: Connection) => {
+        f.call((it, conn))
+      })
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark Streaming dStream foreach
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * @param javaDstream Original DStream with data to iterate over
+   * @param f           Function to be given a iterator to iterate through
+   *                    the JavaDStream values and a HConnection object to
+   *                    interact with HBase
+   */
+  def foreachPartition[T](javaDstream: JavaDStream[T],
+                          f: VoidFunction[(Iterator[T], Connection)]) = {
+    hbaseContext.foreachPartition(javaDstream.dstream,
+      (it: Iterator[T], conn: Connection) => f.call((it, conn)))
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark JavaRDD mapPartition.
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * Note: Make sure to partition correctly to avoid memory issue when
+   * getting data from HBase
+   *
+   * @param javaRdd Original JavaRdd with data to iterate over
+   * @param f       Function to be given a iterator to iterate through
+   *                the RDD values and a HConnection object to interact
+   *                with HBase
+   * @return        Returns a new RDD generated by the user definition
+   *                function just like normal mapPartition
+   */
+  def mapPartitions[T, R](javaRdd: JavaRDD[T],
+                          f: FlatMapFunction[(java.util.Iterator[T],
+                            Connection), R]): JavaRDD[R] = {
+
+    def fn = (it: Iterator[T], conn: Connection) =>
+      asScalaIterator(
+        f.call((asJavaIterator(it), conn))
+      )
+
+    JavaRDD.fromRDD(hbaseContext.mapPartitions(javaRdd.rdd,
+      (iterator: Iterator[T], connection: Connection) =>
+        fn(iterator, connection))(fakeClassTag[R]))(fakeClassTag[R])
+  }
+
+  /**
+   * A simple enrichment of the traditional Spark Streaming JavaDStream
+   * mapPartition.
+   *
+   * This function differs from the original in that it offers the
+   * developer access to a already connected HConnection object
+   *
+   * Note: Do not close the HConnection object.  All HConnection
+   * management is handled outside this method
+   *
+   * Note: Make sure to partition correctly to avoid memory issue when
+   * getting data from HBase
+   *
+   * @param javaDstream Original JavaDStream with data to iterate over
+   * @param mp          Function to be given a iterator to iterate through
+   *                    the JavaDStream values and a HConnection object to
+   *                    interact with HBase
+   * @return            Returns a new JavaDStream generated by the user
+   *                    definition function just like normal mapPartition
+   */
+  def streamMap[T, U](javaDstream: JavaDStream[T],
+                      mp: Function[(Iterator[T], Connection), Iterator[U]]):
+  JavaDStream[U] = {
+    JavaDStream.fromDStream(hbaseContext.streamMapPartitions(javaDstream.dstream,
+      (it: Iterator[T], conn: Connection) =>
+        mp.call((it, conn)))(fakeClassTag[U]))(fakeClassTag[U])
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.foreachPartition method.
+   *
+   * It allow addition support for a user to take JavaRDD
+   * and generate puts and send them to HBase.
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param javaRdd   Original JavaRDD with data to iterate over
+   * @param tableName The name of the table to put into
+   * @param f         Function to convert a value in the JavaRDD
+   *                  to a HBase Put
+   */
+  def bulkPut[T](javaRdd: JavaRDD[T],
+                 tableName: TableName,
+                 f: Function[(T), Put]) {
+
+    hbaseContext.bulkPut(javaRdd.rdd, tableName, (t: T) => f.call(t))
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamMapPartition method.
+   *
+   * It allow addition support for a user to take a JavaDStream and
+   * generate puts and send them to HBase.
+   *
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param javaDstream Original DStream with data to iterate over
+   * @param tableName   The name of the table to put into
+   * @param f           Function to convert a value in
+   *                    the JavaDStream to a HBase Put
+   */
+  def streamBulkPut[T](javaDstream: JavaDStream[T],
+                       tableName: TableName,
+                       f: Function[T, Put]) = {
+    hbaseContext.streamBulkPut(javaDstream.dstream,
+      tableName,
+      (t: T) => f.call(t))
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.foreachPartition method.
+   *
+   * It allow addition support for a user to take a JavaRDD and
+   * generate delete and send them to HBase.
+   *
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param javaRdd   Original JavaRDD with data to iterate over
+   * @param tableName The name of the table to delete from
+   * @param f         Function to convert a value in the JavaRDD to a
+   *                  HBase Deletes
+   * @param batchSize The number of deletes to batch before sending to HBase
+   */
+  def bulkDelete[T](javaRdd: JavaRDD[T], tableName: TableName,
+                    f: Function[T, Delete], batchSize: Integer) {
+    hbaseContext.bulkDelete(javaRdd.rdd, tableName, (t: T) => f.call(t), batchSize)
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamBulkMutation method.
+   *
+   * It allow addition support for a user to take a JavaDStream and
+   * generate Delete and send them to HBase.
+   *
+   * The complexity of managing the HConnection is
+   * removed from the developer
+   *
+   * @param javaDStream Original DStream with data to iterate over
+   * @param tableName   The name of the table to delete from
+   * @param f           Function to convert a value in the JavaDStream to a
+   *                    HBase Delete
+   * @param batchSize   The number of deletes to be sent at once
+   */
+  def streamBulkDelete[T](javaDStream: JavaDStream[T],
+                          tableName: TableName,
+                          f: Function[T, Delete],
+                          batchSize: Integer) = {
+    hbaseContext.streamBulkDelete(javaDStream.dstream, tableName,
+      (t: T) => f.call(t),
+      batchSize)
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.mapPartition method.
+   *
+   * It allow addition support for a user to take a JavaRDD and generates a
+   * new RDD based on Gets and the results they bring back from HBase
+   *
+   * @param tableName     The name of the table to get from
+   * @param batchSize     batch size of how many gets to retrieve in a single fetch
+   * @param javaRdd       Original JavaRDD with data to iterate over
+   * @param makeGet       Function to convert a value in the JavaRDD to a
+   *                      HBase Get
+   * @param convertResult This will convert the HBase Result object to
+   *                      what ever the user wants to put in the resulting
+   *                      JavaRDD
+   * @return              New JavaRDD that is created by the Get to HBase
+   */
+  def bulkGet[T, U](tableName: TableName,
+                    batchSize: Integer,
+                    javaRdd: JavaRDD[T],
+                    makeGet: Function[T, Get],
+                    convertResult: Function[Result, U]): JavaRDD[U] = {
+
+    JavaRDD.fromRDD(hbaseContext.bulkGet[T, U](tableName,
+      batchSize,
+      javaRdd.rdd,
+      (t: T) => makeGet.call(t),
+      (r: Result) => {
+        convertResult.call(r)
+      })(fakeClassTag[U]))(fakeClassTag[U])
+
+  }
+
+  /**
+   * A simple abstraction over the HBaseContext.streamMap method.
+   *
+   * It allow addition support for a user to take a DStream and
+   * generates a new DStream based on Gets and the results
+   * they bring back from HBase
+   *
+
+   * @param tableName     The name of the table to get from
+   * @param batchSize     The number of gets to be batched together
+   * @param javaDStream   Original DStream with data to iterate over
+   * @param makeGet       Function to convert a value in the JavaDStream to a
+   *                      HBase Get
+   * @param convertResult This will convert the HBase Result object to
+   *                      what ever the user wants to put in the resulting
+   *                      JavaDStream
+   * @return              New JavaDStream that is created by the Get to HBase
+   */
+  def streamBulkGet[T, U](tableName: TableName,
+                          batchSize: Integer,
+                          javaDStream: JavaDStream[T],
+                          makeGet: Function[T, Get],
+                          convertResult: Function[Result, U]) {
+    JavaDStream.fromDStream(hbaseContext.streamBulkGet(tableName,
+      batchSize,
+      javaDStream.dstream,
+      (t: T) => makeGet.call(t),
+      (r: Result) => convertResult.call(r))(fakeClassTag[U]))(fakeClassTag[U])
+  }
+
+  /**
+   * This function will use the native HBase TableInputFormat with the
+   * given scan object to generate a new JavaRDD
+   *
+   * @param tableName The name of the table to scan
+   * @param scans     The HBase scan object to use to read data from HBase
+   * @param f         Function to convert a Result object from HBase into
+   *                  What the user wants in the final generated JavaRDD
+   * @return          New JavaRDD with results from scan
+   */
+  def hbaseRDD[U](tableName: TableName,
+                  scans: Scan,
+                  f: Function[(ImmutableBytesWritable, Result), U]):
+  JavaRDD[U] = {
+    JavaRDD.fromRDD(
+      hbaseContext.hbaseRDD[U](tableName,
+        scans,
+        (v: (ImmutableBytesWritable, Result)) =>
+          f.call((v._1, v._2)))(fakeClassTag[U]))(fakeClassTag[U])
+  }
+
+  /**
+   * A overloaded version of HBaseContext hbaseRDD that define the
+   * type of the resulting JavaRDD
+   *
+   * @param tableName The name of the table to scan
+   * @param scans     The HBase scan object to use to read data from HBase
+   * @return          New JavaRDD with results from scan
+   */
+  def hbaseRDD(tableName: TableName,
+               scans: Scan):
+  JavaRDD[(ImmutableBytesWritable, Result)] = {
+    JavaRDD.fromRDD(hbaseContext.hbaseRDD(tableName, scans))
+  }
+
+  /**
+   * Produces a ClassTag[T], which is actually just a casted ClassTag[AnyRef].
+   *
+   * This method is used to keep ClassTags out of the external Java API, as the Java compiler
+   * cannot produce them automatically. While this ClassTag-faking does please the compiler,
+   * it can cause problems at runtime if the Scala API relies on ClassTags for correctness.
+   *
+   * Often, though, a ClassTag[AnyRef] will not lead to incorrect behavior,
+   * just worse performance or security issues.
+   * For instance, an Array[AnyRef] can hold any type T,
+   * but may lose primitive
+   * specialization.
+   */
+  private[spark]
+  def fakeClassTag[T]: ClassTag[T] = ClassTag.AnyRef.asInstanceOf[ClassTag[T]]
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/KeyFamilyQualifier.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/KeyFamilyQualifier.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/KeyFamilyQualifier.scala
new file mode 100644
index 0000000..1e05340
--- /dev/null
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/spark/KeyFamilyQualifier.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.s2graph.s2jobs.spark
+
+import java.io.Serializable
+
+import org.apache.hadoop.hbase.util.Bytes
+
+/**
+ * This is the key to be used for sorting and shuffling.
+ *
+ * We will only partition on the rowKey but we will sort on all three
+ *
+ * @param rowKey    Record RowKey
+ * @param family    Record ColumnFamily
+ * @param qualifier Cell Qualifier
+ */
+class KeyFamilyQualifier(val rowKey:Array[Byte], val family:Array[Byte], val qualifier:Array[Byte])
+  extends Comparable[KeyFamilyQualifier] with Serializable {
+  override def compareTo(o: KeyFamilyQualifier): Int = {
+    var result = Bytes.compareTo(rowKey, o.rowKey)
+    if (result == 0) {
+      result = Bytes.compareTo(family, o.family)
+      if (result == 0) result = Bytes.compareTo(qualifier, o.qualifier)
+    }
+    result
+  }
+  override def toString: String = {
+    Bytes.toString(rowKey) + ":" + Bytes.toString(family) + ":" + Bytes.toString(qualifier)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/db7f0191/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/S2GraphHelperTest.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/S2GraphHelperTest.scala b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/S2GraphHelperTest.scala
new file mode 100644
index 0000000..6e89466
--- /dev/null
+++ b/s2jobs/src/test/scala/org/apache/s2graph/s2jobs/S2GraphHelperTest.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.s2graph.s2jobs
+
+class S2GraphHelperTest {
+
+}


[4/5] incubator-s2graph git commit: test s2jobs/loader.py on local.

Posted by st...@apache.org.
test s2jobs/loader.py on local.


Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/041a1375
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/041a1375
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/041a1375

Branch: refs/heads/master
Commit: 041a1375d42efe85a2ce8bc2b596aefcbb4d1c96
Parents: db7f019
Author: DO YUNG YOON <st...@apache.org>
Authored: Tue Mar 6 20:18:03 2018 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Tue Mar 6 20:18:03 2018 +0900

----------------------------------------------------------------------
 s2jobs/loader.py                                | 25 +++++++++++---------
 .../s2jobs/loader/GraphFileGenerator.scala      |  1 +
 .../s2graph/s2jobs/loader/HFileGenerator.scala  |  3 ++-
 3 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/041a1375/s2jobs/loader.py
----------------------------------------------------------------------
diff --git a/s2jobs/loader.py b/s2jobs/loader.py
index d3a9e67..69cfb4a 100644
--- a/s2jobs/loader.py
+++ b/s2jobs/loader.py
@@ -35,6 +35,7 @@ def hfile(args):
 --conf "spark.driver.extraClassPath=%s" \
 --jars %s \
 --master local[2] \
+--deploy-mode client \
 --num-executors %s \
 --driver-memory 1g \
 --executor-memory 2g \
@@ -42,13 +43,14 @@ def hfile(args):
 %s \
 --input %s \
 --tempDir %s \
---output /tmp/%s \
+--output %s \
 --zkQuorum %s \
 --table %s \
 --dbUrl '%s' \
 --dbUser %s \
 --dbPassword %s \
 --dbDriver %s \
+--method SPARK \
 --maxHFilePerRegionServer %s \
 --labelMapping %s \
 --autoEdgeCreate %s""" % (args["HADOOP_CONF_DIR"],
@@ -59,7 +61,7 @@ def hfile(args):
 						  JAR,
 						  args["input"],
 						  args["tempDir"],
-						  args["htable_name"],
+                          args["output"],
 						  args["hbase_zk"],
 						  args["htable_name"],
 						  args["db_url"],
@@ -89,8 +91,8 @@ def chmod(args):
 	return ret
 
 def load(args):
-	cmd = "export HADOOP_CONF_DIR=%s; export HBASE_CONF_DIR=%s; hbase %s /tmp/%s %s" % \
-		  (args["HADOOP_CONF_DIR"], args["HBASE_CONF_DIR"], LOADER_CLASS, args["htable_name"], args["htable_name"])
+	cmd = "export HADOOP_CONF_DIR=%s; export HBASE_CONF_DIR=%s; hbase %s %s %s" % \
+		  (args["HADOOP_CONF_DIR"], args["HBASE_CONF_DIR"], LOADER_CLASS, args["output"], args["htable_name"])
 	print(cmd)
 	ret = os.system(cmd)
 	print(cmd, "return", ret)
@@ -104,8 +106,8 @@ def run(args):
 	send("[Start]: bulk loader")
 	ret = hfile(args)
 
-	if ret != 0: return send("[Failed]: loader build hfile failed %s" % ret)
-	else: send("[Success]: loader build hfile")
+	# if ret != 0: return send("[Failed]: loader build hfile failed %s" % ret)
+	# else: send("[Success]: loader build hfile")
 
 	# ret = distcp(args)
 	#
@@ -119,8 +121,8 @@ def run(args):
 
 	ret = load(args)
 
-	if ret != 0: return send("[Failed]: loader complete bulkload failed %s" % ret)
-	else: send("[Success]: loader complete bulkload")
+	# if ret != 0: return send("[Failed]: loader complete bulkload failed %s" % ret)
+	# else: send("[Success]: loader complete bulkload")
 
 
 LOADER_CLASS = "org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles"
@@ -132,8 +134,8 @@ DB_URL="jdbc:mysql://localhost:3306/graph_dev"
 # DB_URL="jdbc:h2:file:./var/metastore;MODE=MYSQL"
 args = {
 	"HADOOP_CONF_DIR": "/usr/local/Cellar/hadoop/2.7.3/libexec/etc/hadoop",
-	"HBASE_CONF_DIR": "/usr/local/opt/hbase/libexec/conf",
-	"htable_name": "test",
+	"HBASE_CONF_DIR": "/usr/local/Cellar/hbase/1.2.6/libexec/conf",
+	"htable_name": "s2graph",
 	"hbase_namenode": "hdfs://localhost:8020",
 	"hbase_zk": "localhost",
 	"db_driver": DB_DRIVER,
@@ -147,7 +149,8 @@ args = {
 	"-bandwidth": 10,
 	"num_executors": 2,
 	"input": "/tmp/imei-20.txt",
-	"tempDir": "/tmp/bulkload_tmp"
+	"tempDir": "/tmp/bulkload_tmp",
+    "output": "/tmp/bulkload_output"
 }
 
 run(args)

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/041a1375/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
index 51476c1..79eca36 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/GraphFileGenerator.scala
@@ -38,5 +38,6 @@ object GraphFileGenerator {
       case "SPARK" => HFileGenerator.generate(sc, s2Config, input, options)
       case _ => throw new IllegalArgumentException("only supported type is MR/SPARK.")
     }
+    System.exit(0)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/041a1375/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
----------------------------------------------------------------------
diff --git a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
index 9c3de2a..acd3886 100644
--- a/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
+++ b/s2jobs/src/main/scala/org/apache/s2graph/s2jobs/loader/HFileGenerator.scala
@@ -32,6 +32,7 @@ import org.apache.s2graph.core._
 import org.apache.s2graph.core.mysqls.{Label, LabelMeta}
 import org.apache.s2graph.core.storage.hbase.AsynchbaseStorageManagement
 import org.apache.s2graph.core.types.{InnerValLikeWithTs, SourceVertexId}
+import org.apache.s2graph.core.utils.logger
 import org.apache.s2graph.s2jobs.S2GraphHelper
 import org.apache.s2graph.s2jobs.spark._
 import org.apache.spark.SparkContext
@@ -131,7 +132,7 @@ object HFileGenerator extends RawFileGenerator {
 
     hbaseConf.set("hbase.zookeeper.quorum", graphFileOptions.zkQuorum)
     hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, graphFileOptions.tableName)
-    hbaseConf.set("hadoop.tmp.dir", s"/tmp/${graphFileOptions.tableName}")
+//    hbaseConf.set("hadoop.tmp.dir", s"/tmp/${graphFileOptions.tableName}")
 
     hbaseConf
   }