You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@s2graph.apache.org by st...@apache.org on 2017/07/31 01:05:37 UTC
[23/25] incubator-s2graph git commit: merge S2GRAPH-152.
merge S2GRAPH-152.
Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/5cc41aa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/5cc41aa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/5cc41aa4
Branch: refs/heads/master
Commit: 5cc41aa4bc66bf07f3c67813a0efb0c5c155fd50
Parents: d8de89d
Author: DO YUNG YOON <st...@apache.org>
Authored: Sun Jul 30 11:07:47 2017 +0900
Committer: DO YUNG YOON <st...@apache.org>
Committed: Sun Jul 30 11:07:47 2017 +0900
----------------------------------------------------------------------
CHANGES | 2 +
dev_support/graph_mysql/schema.sql | 7 ++-
.../org/apache/s2graph/core/mysqls/schema.sql | 5 +-
.../org/apache/s2graph/core/Management.scala | 22 ++++----
.../s2graph/core/index/IndexProvider.scala | 8 +--
.../s2graph/core/mysqls/GlobalIndex.scala | 38 ++++++++------
.../s2graph/core/index/IndexProviderTest.scala | 2 +-
.../s2graph/core/models/GlobalIndexTest.scala | 55 +++++++++++---------
.../core/tinkerpop/S2GraphProvider.scala | 7 ++-
.../core/tinkerpop/structure/S2GraphTest.scala | 2 +-
10 files changed, 84 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 35ed7d4..4345362 100644
--- a/CHANGES
+++ b/CHANGES
@@ -242,6 +242,8 @@ Release 0.1.0 - unreleased
S2GRAPH-131: Add actual implementation on interfaces from TinkerPop3 structure package. (Committed by DOYUNG YOON).
S2GRAPH-136: Validate TinkerPop3 interface with gremlin-test suite. (Committed by DOYUNG YOON).
+
+ S2GRAPH-152: Add buildGlobalIndex API on Management. (Committed by DOYUNG YOON).
TEST
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/dev_support/graph_mysql/schema.sql
----------------------------------------------------------------------
diff --git a/dev_support/graph_mysql/schema.sql b/dev_support/graph_mysql/schema.sql
index 58c4328..ea72c63 100644
--- a/dev_support/graph_mysql/schema.sql
+++ b/dev_support/graph_mysql/schema.sql
@@ -120,22 +120,21 @@ CREATE TABLE `labels` (
ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id);
-
-- ----------------------------
-- Table structure for `global_index`
-- ----------------------------
DROP TABLE IF EXISTS `global_indices`;
CREATE TABLE `global_indices` (
`id` integer NOT NULL AUTO_INCREMENT,
+ `element_type` varchar(64) NOT NULL,
`prop_names` varchar(255) NOT NULL,
`index_name` varchar(64) NOT NULL,
PRIMARY KEY (`id`),
- UNIQUE KEY `ux_global_index_index_name` (`index_name`),
- UNIQUE KEY `ux_global_index_prop_names` (`prop_names`)
+ UNIQUE KEY `ux_global_index_element_type_index_name` (`element_type`, `index_name`),
+ UNIQUE KEY `ux_global_index_element_type_prop_names` (`element_type`, `prop_names`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-
-- ----------------------------
-- Table structure for `label_metas`
-- ----------------------------
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
----------------------------------------------------------------------
diff --git a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
index 521c9d2..f5f6a61 100644
--- a/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
+++ b/s2core/src/main/resources/org/apache/s2graph/core/mysqls/schema.sql
@@ -115,11 +115,12 @@ ALTER TABLE labels add FOREIGN KEY(service_id) REFERENCES services(id);
DROP TABLE IF EXISTS `global_indices`;
CREATE TABLE `global_indices` (
`id` integer NOT NULL AUTO_INCREMENT,
+ `element_type` varchar(64) NOT NULL,
`prop_names` varchar(255) NOT NULL,
`index_name` varchar(64) NOT NULL,
PRIMARY KEY (`id`),
- UNIQUE KEY `ux_global_index_index_name` (`index_name`),
- UNIQUE KEY `ux_global_index_prop_names` (`prop_names`)
+ UNIQUE KEY `ux_global_index_element_type_index_name` (`element_type`, `index_name`),
+ UNIQUE KEY `ux_global_index_element_type_prop_names` (`element_type`, `prop_names`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
index 57791af..13610b4 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/Management.scala
@@ -349,15 +349,19 @@ class Management(graph: S2Graph) {
old.consistencyLevel, hTableName, old.hTableTTL, old.schemaVersion, old.isAsync, old.compressionAlgorithm, old.options)
}
- def buildGlobalIndex(name: String, propNames: Seq[String]): GlobalIndex = {
- Model.withTx { implicit session =>
- GlobalIndex.findBy(name, false) match {
- case None =>
- GlobalIndex.insert(name, propNames)
- GlobalIndex.findBy(name, false).get
- case Some(oldIndex) => oldIndex
- }
- }.get
+ def buildGlobalVertexIndex(name: String, propNames: Seq[String]): GlobalIndex =
+ buildGlobalIndex(GlobalIndex.VertexType, name, propNames)
+
+ def buildGlobalEdgeIndex(name: String, propNames: Seq[String]): GlobalIndex =
+ buildGlobalIndex(GlobalIndex.EdgeType, name, propNames)
+
+ def buildGlobalIndex(elementType: String, name: String, propNames: Seq[String]): GlobalIndex = {
+ GlobalIndex.findBy(elementType, name, false) match {
+ case None =>
+ GlobalIndex.insert(elementType, name, propNames)
+ GlobalIndex.findBy(elementType, name, false).get
+ case Some(oldIndex) => oldIndex
+ }
}
def getCurrentStorageInfo(labelName: String): Try[Map[String, String]] = for {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
index 57f384c..b52ffda 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/index/IndexProvider.scala
@@ -205,7 +205,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
}
override def mutateVertices(vertices: Seq[S2Vertex]): Seq[Boolean] = {
- val globalIndexOptions = GlobalIndex.findAll()
+ val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.VertexType)
globalIndexOptions.map { globalIndex =>
val writer = getOrElseCreateIndexWriter(globalIndex.indexName)
@@ -223,7 +223,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
}
override def mutateEdges(edges: Seq[S2Edge]): Seq[Boolean] = {
- val globalIndexOptions = GlobalIndex.findAll()
+ val globalIndexOptions = GlobalIndex.findAll(GlobalIndex.EdgeType)
globalIndexOptions.map { globalIndex =>
val writer = getOrElseCreateIndexWriter(globalIndex.indexName)
@@ -244,7 +244,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
val field = eidField
val ids = new java.util.HashSet[EdgeId]
- GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex =>
+ GlobalIndex.findGlobalIndex(GlobalIndex.EdgeType, hasContainers).map { globalIndex =>
val queryString = buildQueryString(hasContainers)
try {
@@ -277,7 +277,7 @@ class LuceneIndexProvider(config: Config) extends IndexProvider {
val field = vidField
val ids = new java.util.HashSet[VertexId]
- GlobalIndex.findGlobalIndex(hasContainers).map { globalIndex =>
+ GlobalIndex.findGlobalIndex(GlobalIndex.VertexType, hasContainers).map { globalIndex =>
val queryString = buildQueryString(hasContainers)
try {
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala
----------------------------------------------------------------------
diff --git a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala
index 347c083..b3657e1 100644
--- a/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala
+++ b/s2core/src/main/scala/org/apache/s2graph/core/mysqls/GlobalIndex.scala
@@ -35,35 +35,40 @@ object GlobalIndex extends Model[GlobalIndex] {
val labelField = "_label_"
val serviceField = "_service_"
val serviceColumnField = "_serviceColumn_"
-
+ val EdgeType = "edge"
+ val VertexType = "vertex"
val hiddenIndexFields = Set(vidField, eidField, labelField, serviceField, serviceColumnField)
- val DefaultIndexName = GlobalIndex(None, Seq(vidField, eidField, serviceField, serviceColumnField, labelField), "_default_")
val TableName = "global_indices"
def apply(rs: WrappedResultSet): GlobalIndex = {
- GlobalIndex(rs.intOpt("id"), rs.string("prop_names").split(",").sorted, rs.string("index_name"))
+ GlobalIndex(rs.intOpt("id"),
+ rs.string("element_type"),
+ rs.string("prop_names").split(",").sorted,
+ rs.string("index_name"))
}
- def findBy(indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = {
- val cacheKey = s"indexName=$indexName"
- lazy val sql = sql"""select * from global_indices where index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply()
+ def findBy(elementType: String, indexName: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Option[GlobalIndex] = {
+ val cacheKey = s"elementType=$elementType:indexName=$indexName"
+ lazy val sql = sql"""select * from global_indices where element_type = ${elementType} and index_name = $indexName""".map { rs => GlobalIndex(rs) }.single.apply()
+
if (useCache) withCache(cacheKey){sql}
else sql
}
- def insert(indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = {
+ def insert(elementType: String, indexName: String, propNames: Seq[String])(implicit session: DBSession = AutoSession): Long = {
val allPropNames = (hiddenIndexFields.toSeq ++ propNames).sorted
- sql"""insert into global_indices(prop_names, index_name) values(${allPropNames.mkString(",")}, $indexName)"""
+ sql"""insert into global_indices(element_type, prop_names, index_name)
+ values($elementType, ${allPropNames.mkString(",")}, $indexName)"""
.updateAndReturnGeneratedKey.apply()
}
- def findAll(useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = {
- lazy val ls = sql"""select * from global_indices """.map { rs => GlobalIndex(rs) }.list.apply
+ def findAll(elementType: String, useCache: Boolean = true)(implicit session: DBSession = AutoSession): Seq[GlobalIndex] = {
+ lazy val ls = sql"""select * from global_indices where element_type = $elementType""".map { rs => GlobalIndex(rs) }.list.apply
if (useCache) {
- listCache.withCache("findAll") {
+ listCache.withCache(s"findAll:elementType=$elementType") {
putsToCache(ls.map { globalIndex =>
- val cacheKey = s"indexName=${globalIndex.indexName}"
+ val cacheKey = s"elementType=${globalIndex.elementType}:indexName=${globalIndex.indexName}"
cacheKey -> globalIndex
})
ls
@@ -73,9 +78,9 @@ object GlobalIndex extends Model[GlobalIndex] {
}
}
- def findGlobalIndex(hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = {
+ def findGlobalIndex(elementType: String, hasContainers: java.util.List[HasContainer])(implicit session: DBSession = AutoSession): Option[GlobalIndex] = {
import scala.collection.JavaConversions._
- val indices = findAll(useCache = true)
+ val indices = findAll(elementType, useCache = true)
val keys = hasContainers.map(_.getKey)
val sorted = indices.map { index =>
@@ -88,6 +93,9 @@ object GlobalIndex extends Model[GlobalIndex] {
}
-case class GlobalIndex(id: Option[Int], propNames: Seq[String], indexName: String) {
+case class GlobalIndex(id: Option[Int],
+ elementType: String,
+ propNames: Seq[String],
+ indexName: String) {
lazy val propNamesSet = propNames.toSet
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
index c84d697..8294a8f 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/index/IndexProviderTest.scala
@@ -34,7 +34,7 @@ class IndexProviderTest extends IntegrateCommon {
val indexProvider = IndexProvider(config)
val numOfTry = 1
- lazy val gIndex = management.buildGlobalIndex("test1", Seq("_timestamp", "weight", "time"))
+ lazy val gIndex = management.buildGlobalIndex(GlobalIndex.EdgeType, "test1", Seq("_timestamp", "weight", "time"))
// test("test vertex write/query") {
// gIndex
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala
index b779806..a732564 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/models/GlobalIndexTest.scala
@@ -36,43 +36,46 @@ class GlobalIndexTest extends FunSuite with Matchers with TestCommonWithModels w
}
test("test buildGlobalIndex.") {
- management.buildGlobalIndex("test_global", Seq("weight", "date", "name"))
+ Seq(GlobalIndex.EdgeType, GlobalIndex.VertexType).foreach { elementType =>
+ management.buildGlobalIndex(elementType, "test_global", Seq("weight", "date", "name"))
+ }
}
test("findGlobalIndex.") {
// (weight: 34) AND (weight: [1 TO 100])
- val idx1 = management.buildGlobalIndex("test-1", Seq("weight", "age", "name"))
- val idx2 = management.buildGlobalIndex("test-2", Seq("gender", "age"))
- val idx3 = management.buildGlobalIndex("test-3", Seq("class"))
+ Seq(GlobalIndex.EdgeType, GlobalIndex.VertexType).foreach { elementType =>
+ val idx1 = management.buildGlobalIndex(elementType, "test-1", Seq("weight", "age", "name"))
+ val idx2 = management.buildGlobalIndex(elementType, "test-2", Seq("gender", "age"))
+ val idx3 = management.buildGlobalIndex(elementType, "test-3", Seq("class"))
- var hasContainers = Seq(
- new HasContainer("weight", P.eq(Int.box(34))),
- new HasContainer("age", P.between(Int.box(1), Int.box(100)))
- )
+ var hasContainers = Seq(
+ new HasContainer("weight", P.eq(Int.box(34))),
+ new HasContainer("age", P.between(Int.box(1), Int.box(100)))
+ )
- GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx1))
+ GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx1))
- hasContainers = Seq(
- new HasContainer("gender", P.eq(Int.box(34))),
- new HasContainer("age", P.eq(Int.box(34))),
- new HasContainer("class", P.eq(Int.box(34)))
- )
+ hasContainers = Seq(
+ new HasContainer("gender", P.eq(Int.box(34))),
+ new HasContainer("age", P.eq(Int.box(34))),
+ new HasContainer("class", P.eq(Int.box(34)))
+ )
- GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx2))
+ GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx2))
- hasContainers = Seq(
- new HasContainer("class", P.eq(Int.box(34))),
- new HasContainer("_", P.eq(Int.box(34)))
- )
+ hasContainers = Seq(
+ new HasContainer("class", P.eq(Int.box(34))),
+ new HasContainer("_", P.eq(Int.box(34)))
+ )
- GlobalIndex.findGlobalIndex(hasContainers) shouldBe(Option(idx3))
+ GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(Option(idx3))
- hasContainers = Seq(
- new HasContainer("key", P.eq(Int.box(34))),
- new HasContainer("value", P.eq(Int.box(34)))
- )
-
- GlobalIndex.findGlobalIndex(hasContainers) shouldBe(None)
+ hasContainers = Seq(
+ new HasContainer("key", P.eq(Int.box(34))),
+ new HasContainer("value", P.eq(Int.box(34)))
+ )
+ GlobalIndex.findGlobalIndex(elementType, hasContainers) shouldBe(None)
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
index d8367d7..8bcac9c 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/S2GraphProvider.scala
@@ -24,13 +24,14 @@ import org.apache.s2graph.core.GraphExceptions.LabelNotExistException
import org.apache.s2graph.core.Management.JsonModel.Prop
import org.apache.s2graph.core.S2Graph.{DefaultColumnName, DefaultServiceName}
import org.apache.s2graph.core._
-import org.apache.s2graph.core.mysqls.{ColumnMeta, Label, Service, ServiceColumn}
+import org.apache.s2graph.core.mysqls._
import org.apache.s2graph.core.types.{HBaseType, InnerVal, VertexId}
import org.apache.s2graph.core.utils.logger
import org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData
import org.apache.tinkerpop.gremlin.structure.{Element, Graph, T}
import org.apache.tinkerpop.gremlin.{AbstractGraphProvider, LoadGraphWith}
import java.util
+
import scala.collection.JavaConverters._
object S2GraphProvider {
@@ -508,7 +509,9 @@ class S2GraphProvider extends AbstractGraphProvider {
options = Option("""{"skipReverse": false}""")
)
- val globalIndex = mnt.buildGlobalIndex("global", allProps.map(_.name).toSeq)
+ Seq(GlobalIndex.VertexType, GlobalIndex.EdgeType).foreach { elementType =>
+ mnt.buildGlobalIndex(elementType, "global", allProps.map(_.name).toSeq)
+ }
super.loadGraphData(graph, loadGraphWith, testClass, testName)
}
http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/5cc41aa4/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
----------------------------------------------------------------------
diff --git a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
index badfbfe..34769da 100644
--- a/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
+++ b/s2core/src/test/scala/org/apache/s2graph/core/tinkerpop/structure/S2GraphTest.scala
@@ -41,7 +41,7 @@ class S2GraphTest extends FunSuite with Matchers with TestCommonWithModels {
initTests()
val g = new S2Graph(config)
- lazy val gIndex = management.buildGlobalIndex("S2GraphTest2", Seq("weight"))
+ lazy val gIndex = management.buildGlobalIndex(GlobalIndex.EdgeType, "S2GraphTest2", Seq("weight"))
def printEdges(edges: Seq[Edge]): Unit = {
edges.foreach { edge =>
logger.debug(s"[FetchedEdge]: $edge")