You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2017/03/01 14:32:59 UTC
[16/50] [abbrv] ignite git commit: Added Spatial Index segmentation.
Added Spatial Index segmentation.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e69d3c34
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e69d3c34
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e69d3c34
Branch: refs/heads/master
Commit: e69d3c34915f2f7de1b66f22e0584b7c09d8fe71
Parents: 64ba13b
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Tue Feb 21 11:35:34 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Tue Feb 21 11:52:58 2017 +0300
----------------------------------------------------------------------
.../query/h2/opt/GridH2SpatialIndex.java | 69 +++++++++++++++++---
.../h2/GridH2IndexingSegmentedGeoSelfTest.java | 30 +++++++++
.../testsuites/GeoSpatialIndexingTestSuite.java | 2 +
.../processors/query/h2/IgniteH2Indexing.java | 16 ++---
.../query/h2/opt/GridH2IndexBase.java | 21 +++---
.../query/h2/opt/GridH2TreeIndex.java | 35 ++--------
6 files changed, 118 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
index 3062d13..096b82d 100644
--- a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
+++ b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java
@@ -41,7 +41,6 @@ import org.h2.mvstore.rtree.SpatialKey;
import org.h2.result.SearchRow;
import org.h2.result.SortOrder;
import org.h2.table.IndexColumn;
-import org.h2.table.Table;
import org.h2.table.TableFilter;
import org.h2.value.Value;
import org.h2.value.ValueGeometry;
@@ -67,7 +66,7 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
private boolean closed;
/** */
- private final MVRTreeMap<Long> treeMap;
+ private final MVRTreeMap<Long>[] segments;
/** */
private final Map<Long, GridH2Row> idToRow = new HashMap<>();
@@ -83,7 +82,17 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
* @param idxName Index name.
* @param cols Columns.
*/
- public GridH2SpatialIndex(Table tbl, String idxName, IndexColumn... cols) {
+ public GridH2SpatialIndex(GridH2Table tbl, String idxName, IndexColumn... cols) {
+ this(tbl, idxName, 1, cols);
+ }
+
+ /**
+ * @param tbl Table.
+ * @param idxName Index name.
+ * @param segmentsCnt Index segments count.
+ * @param cols Columns.
+ */
+ public GridH2SpatialIndex(GridH2Table tbl, String idxName, int segmentsCnt, IndexColumn... cols) {
if (cols.length > 1)
throw DbException.getUnsupportedException("can only do one column");
@@ -107,7 +116,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
// Index in memory
store = MVStore.open(null);
- treeMap = store.openMap("spatialIndex", new MVRTreeMap.Builder<Long>());
+
+ segments = new MVRTreeMap[segmentsCnt];
+
+ for (int i = 0; i < segmentsCnt; i++)
+ segments[i] = store.openMap("spatialIndex-" + i, new MVRTreeMap.Builder<Long>());
}
/**
@@ -119,6 +132,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/** {@inheritDoc} */
+ @Override protected int segmentsCount() {
+ return segments.length;
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override protected Object doTakeSnapshot() {
return null; // TODO We do not support snapshots, but probably this is possible.
}
@@ -140,20 +158,26 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
Long rowId = keyToId.get(key);
+ int seg;
+
if (rowId != null) {
- Long oldRowId = treeMap.remove(getEnvelope(idToRow.get(rowId), rowId));
+ seg = segmentForRowID(rowId);
+
+ Long oldRowId = segments[seg].remove(getEnvelope(idToRow.get(rowId), rowId));
assert rowId.equals(oldRowId);
}
else {
rowId = ++rowIds;
+ seg = segmentForRowID(rowId);
+
keyToId.put(key, rowId);
}
GridH2Row old = idToRow.put(rowId, row);
- treeMap.put(getEnvelope(row, rowId), rowId);
+ segments[seg].put(getEnvelope(row, rowId), rowId);
if (old == null)
rowCnt++; // No replace.
@@ -166,6 +190,17 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
}
/**
+ * @param id Row ID.
+ *
+ * @return Segment ID for given row ID.
+ */
+ private int segmentForRowID(Long id) {
+ assert id != null;
+
+ return (int)(id % segmentsCount());
+ }
+
+ /**
* @param row Row.
* @param rowId Row id.
* @return Envelope.
@@ -200,7 +235,9 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
assert oldRow != null;
- if (!treeMap.remove(getEnvelope(row, rowId), rowId))
+ int seg = segmentForRowID(rowId);
+
+ if (!segments[seg].remove(getEnvelope(row, rowId), rowId))
throw DbException.throwInternalError("row not found");
rowCnt--;
@@ -258,7 +295,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
try {
checkClosed();
- return new GridH2Cursor(rowIterator(treeMap.keySet().iterator(), filter));
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ return new GridH2Cursor(rowIterator(segment.keySet().iterator(), filter));
}
finally {
l.unlock();
@@ -305,7 +346,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
if (!first)
throw DbException.throwInternalError("Spatial Index can only be fetch by ascending order");
- Iterator<GridH2Row> iter = rowIterator(treeMap.keySet().iterator(), null);
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ Iterator<GridH2Row> iter = rowIterator(segment.keySet().iterator(), null);
return new SingleRowCursor(iter.hasNext() ? iter.next() : null);
}
@@ -334,7 +379,11 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex
if (intersection == null)
return find(filter.getSession(), null, null);
- return new GridH2Cursor(rowIterator(treeMap.findIntersectingKeys(getEnvelope(intersection, 0)), filter));
+ final int seg = threadLocalSegment();
+
+ final MVRTreeMap<Long> segment = segments[seg];
+
+ return new GridH2Cursor(rowIterator(segment.findIntersectingKeys(getEnvelope(intersection, 0)), filter));
}
finally {
l.unlock();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
new file mode 100644
index 0000000..b806321
--- /dev/null
+++ b/modules/geospatial/src/test/java/org/apache/ignite/internal/processors/query/h2/GridH2IndexingSegmentedGeoSelfTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ * Test for segmented geo index.
+ */
+public class GridH2IndexingSegmentedGeoSelfTest extends GridH2IndexingGeoSelfTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration cacheConfiguration(String gridName) throws Exception {
+ return super.cacheConfiguration(gridName).setQueryParallelism(7);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java b/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
index 1773894..3907b9e 100644
--- a/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
+++ b/modules/geospatial/src/test/java/org/apache/ignite/testsuites/GeoSpatialIndexingTestSuite.java
@@ -20,6 +20,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.query.h2.GridBinaryH2IndexingGeoSelfTest;
import org.apache.ignite.internal.processors.query.h2.GridH2IndexingGeoSelfTest;
+import org.apache.ignite.internal.processors.query.h2.GridH2IndexingSegmentedGeoSelfTest;
/**
* Geospatial indexing tests.
@@ -35,6 +36,7 @@ public class GeoSpatialIndexingTestSuite extends TestSuite {
// Geo.
suite.addTestSuite(GridH2IndexingGeoSelfTest.class);
suite.addTestSuite(GridBinaryH2IndexingGeoSelfTest.class);
+ suite.addTestSuite(GridH2IndexingSegmentedGeoSelfTest.class);
return suite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 2f40d87..652d1f3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2813,7 +2813,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @param cols Columns.
*/
private SpatialIndex createH2SpatialIndex(
- Table tbl,
+ GridH2Table tbl,
String idxName,
IndexColumn[] cols
) {
@@ -2823,14 +2823,17 @@ public class IgniteH2Indexing implements GridQueryIndexing {
Class<?> cls = Class.forName(className);
Constructor<?> ctor = cls.getConstructor(
- Table.class,
+ GridH2Table.class,
String.class,
+ Integer.TYPE,
IndexColumn[].class);
if (!ctor.isAccessible())
ctor.setAccessible(true);
- return (SpatialIndex)ctor.newInstance(tbl, idxName, cols);
+ final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
+
+ return (SpatialIndex)ctor.newInstance(tbl, idxName, segments, cols);
}
catch (Exception e) {
throw new IgniteException("Failed to instantiate: " + className, e);
@@ -2845,12 +2848,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
* @return
*/
private Index createTreeIndex(String idxName, GridH2Table tbl, boolean pk, List<IndexColumn> columns) {
- GridCacheContext<?, ?> cctx = tbl.rowDescriptor().context();
-
- if (cctx != null && cctx.config().getQueryParallelism() > 1)
- return new GridH2TreeIndex(idxName, tbl, pk, columns, cctx.config().getQueryParallelism());
+ final int segments = tbl.rowDescriptor().context().config().getQueryParallelism();
- return new GridH2TreeIndex(idxName, tbl, pk, columns, 1);
+ return new GridH2TreeIndex(idxName, tbl, pk, columns, segments);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 131e03b..89d661d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -183,7 +183,15 @@ public abstract class GridH2IndexBase extends BaseIndex {
* @return Index segment ID for current query context.
*/
protected int threadLocalSegment() {
- return 0;
+ GridH2QueryContext qctx = GridH2QueryContext.get();
+
+ if(segmentsCount() == 1)
+ return 0;
+
+ if(qctx == null)
+ throw new IllegalStateException("GridH2QueryContext is not initialized.");
+
+ return qctx.segment();
}
/**
@@ -669,7 +677,7 @@ public abstract class GridH2IndexBase extends BaseIndex {
throw new GridH2RetryException("Failed to find node.");
}
- return new SegmentKey(node, segment(partition));
+ return new SegmentKey(node, segmentForPartition(partition));
}
/** */
@@ -829,19 +837,16 @@ public abstract class GridH2IndexBase extends BaseIndex {
}
/** @return Index segments count. */
- protected int segmentsCount() {
- return 1;
- }
+ protected abstract int segmentsCount();
/**
* @param partition Partition idx.
* @return Segment ID for given key
*/
- protected int segment(int partition) {
- return 0;
+ protected int segmentForPartition(int partition){
+ return segmentsCount() == 1 ? 0 : (partition % segmentsCount());
}
-
/**
* Simple cursor from a single node.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/e69d3c34/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
index 0829df0..64ca9ea 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2TreeIndex.java
@@ -50,7 +50,6 @@ import org.jetbrains.annotations.Nullable;
*/
@SuppressWarnings("ComparatorNotSerializable")
public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridSearchRowPointer> {
-
/** */
private static Field KEY_FIELD;
@@ -224,20 +223,6 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
super.destroy();
}
-
- /** {@inheritDoc} */
- protected int threadLocalSegment() {
- GridH2QueryContext qctx = GridH2QueryContext.get();
-
- if(segments.length == 1)
- return 0;
-
- if(qctx == null)
- throw new IllegalStateException("GridH2QueryContext is not initialized.");
-
- return qctx.segment();
- }
-
/** {@inheritDoc} */
@Override public long getRowCount(@Nullable Session ses) {
IndexingQueryFilter f = threadLocalFilter();
@@ -433,7 +418,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
/** {@inheritDoc} */
@Override public GridH2Row put(GridH2Row row) {
- int seg = segment(row);
+ int seg = segmentForRow(row);
return segments[seg].put(row, row);
}
@@ -442,7 +427,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
@Override public GridH2Row remove(SearchRow row) {
GridSearchRowPointer comparable = comparable(row, 0);
- int seg = segment(row);
+ int seg = segmentForRow(row);
return segments[seg].remove(comparable);
}
@@ -453,18 +438,10 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
}
/**
- * @param partition Parttition idx.
- * @return index currentSegment Id for given key
- */
- protected int segment(int partition) {
- return partition % segments.length;
- }
-
- /**
- * @param row
- * @return index currentSegment Id for given row
+ * @param row Table row.
+ * @return Segment ID for given row.
*/
- private int segment(SearchRow row) {
+ private int segmentForRow(SearchRow row) {
assert row != null;
CacheObject key;
@@ -491,7 +468,7 @@ public class GridH2TreeIndex extends GridH2IndexBase implements Comparator<GridS
else
key = ctx.toCacheKeyObject(row.getValue(0));
- return segment(ctx.affinity().partition(key));
+ return segmentForPartition(ctx.affinity().partition(key));
}
else
return 0;