You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ki...@apache.org on 2016/04/28 09:44:50 UTC
[3/3] incubator-asterixdb git commit: MBR size optimization for point
data in R-Tree
MBR size optimization for point data in R-Tree
Change-Id: Ic48304a221f462c66bb1417f291b0ce4f2ce7581
Reviewed-on: https://asterix-gerrit.ics.uci.edu/831
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/e964f9b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/e964f9b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/e964f9b8
Branch: refs/heads/master
Commit: e964f9b8083e0ead94a6d5d3b5e38ac685290629
Parents: 3ca85ee
Author: kisskys <ki...@apache.org>
Authored: Wed Apr 27 17:33:11 2016 -0700
Committer: Young-Seok Kim <ki...@gmail.com>
Committed: Thu Apr 28 00:43:37 2016 -0700
----------------------------------------------------------------------
...IntroduceSecondaryIndexInsertDeleteRule.java | 23 ++-
.../external/ExternalIndexingOperations.java | 9 +-
.../file/SecondaryBTreeOperationsHelper.java | 63 +++---
.../file/SecondaryIndexOperationsHelper.java | 29 ++-
.../SecondaryInvertedIndexOperationsHelper.java | 4 +-
.../file/SecondaryRTreeOperationsHelper.java | 198 +++++++++++++------
.../metadata/declared/AqlMetadataProvider.java | 15 +-
.../ExternalRTreeLocalResourceMetadata.java | 9 +-
.../resource/LSMRTreeLocalResourceMetadata.java | 8 +-
.../physical/IndexBulkloadPOperator.java | 10 +-
.../lsm/rtree/LSMRTreeOperatorTestHelper.java | 2 +-
...eWithAntiMatterTuplesOperatorTestHelper.java | 2 +-
.../AbstractLSMRTreeDataflowHelper.java | 16 +-
.../dataflow/ExternalRTreeDataflowHelper.java | 22 ++-
.../ExternalRTreeDataflowHelperFactory.java | 8 +-
.../rtree/dataflow/LSMRTreeDataflowHelper.java | 10 +-
.../dataflow/LSMRTreeDataflowHelperFactory.java | 10 +-
...RTreeWithAntiMatterTuplesDataflowHelper.java | 8 +-
...thAntiMatterTuplesDataflowHelperFactory.java | 13 +-
.../am/lsm/rtree/impls/AbstractLSMRTree.java | 12 +-
.../am/lsm/rtree/impls/ExternalRTree.java | 6 +-
.../storage/am/lsm/rtree/impls/LSMRTree.java | 9 +-
.../impls/LSMRTreeWithAntiMatterTuples.java | 40 ++--
.../am/lsm/rtree/impls/RTreeFactory.java | 8 +-
.../LSMRTreeTupleReferenceForPointMBR.java | 162 +++++++++++++++
.../LSMRTreeTupleWriterFactoryForPointMBR.java | 45 +++++
.../tuples/LSMRTreeTupleWriterForPointMBR.java | 149 ++++++++++++++
.../am/lsm/rtree/utils/LSMRTreeUtils.java | 83 +++++---
.../am/rtree/dataflow/RTreeDataflowHelper.java | 11 +-
.../storage/am/rtree/frames/RTreeNSMFrame.java | 4 +-
.../am/rtree/frames/RTreeNSMInteriorFrame.java | 4 +-
.../frames/RTreeNSMInteriorFrameFactory.java | 8 +-
.../am/rtree/frames/RTreeNSMLeafFrame.java | 4 +-
.../rtree/frames/RTreeNSMLeafFrameFactory.java | 8 +-
.../hyracks/storage/am/rtree/impls/RTree.java | 62 +++---
.../storage/am/rtree/util/RTreeUtils.java | 13 +-
.../am/lsm/rtree/LSMRTreeExamplesTest.java | 7 +-
...SMRTreeWithAntiMatterTuplesExamplesTest.java | 12 +-
.../multithread/LSMRTreeMultiThreadTest.java | 26 +--
...TreeWithAntiMatterTuplesMultiThreadTest.java | 37 ++--
.../am/lsm/rtree/util/LSMRTreeTestContext.java | 9 +-
...LSMRTreeWithAntiMatterTuplesTestContext.java | 15 +-
.../storage/am/rtree/RTreeExamplesTest.java | 7 +-
.../storage/am/rtree/RTreeSearchCursorTest.java | 24 ++-
.../rtree/multithread/RTreeMultiThreadTest.java | 14 +-
.../am/rtree/utils/RTreeTestContext.java | 2 +-
46 files changed, 894 insertions(+), 346 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 8487374..e3cca9b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -100,6 +100,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
FunctionIdentifier fid = null;
/** find the record variable */
InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) op1;
+ boolean isBulkload = insertOp.isBulkload();
ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
LogicalVariable recordVar = null;
List<LogicalVariable> usedRecordVars = new ArrayList<>();
@@ -441,8 +442,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
secondaryKeyFields.get(0), recType);
IAType spatialType = keyPairType.first;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
- int numKeys = dimension * 2;
+ int numKeys = (isPointMBR && isBulkload) ? dimension : dimension * 2;
// Get variables and expressions
List<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
@@ -463,6 +466,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
secondaryExpressions.add(
new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
}
+ if (isPointMBR && isBulkload) {
+ //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and
+ //createFieldPermutationForBulkLoadOp(int) for more details.
+ for (LogicalVariable secondaryKeyVar : keyVarList) {
+ secondaryExpressions.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(secondaryKeyVar)));
+ }
+ }
AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
@@ -490,6 +501,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
new VariableReferenceExpression(secondaryKeyVar)));
}
+ if (isPointMBR && isBulkload) {
+ //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and
+ //createFieldPermutationForBulkLoadOp(int) for more details.
+ for (LogicalVariable secondaryKeyVar : originalKeyVarList) {
+ prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(secondaryKeyVar)));
+ }
+ }
originalAssignCoordinates = new AssignOperator(originalKeyVarList, originalKeyExprList);
originalAssignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
context.computeAndSetTypeEnvironmentForOperator(originalAssignCoordinates);
@@ -654,7 +673,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
@SuppressWarnings("unchecked")
private void prepareVarAndExpression(List<String> fields, String[] recordFields, LogicalVariable recordVar,
List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable> vars, IOptimizationContext context)
- throws AlgebricksException {
+ throws AlgebricksException {
// Get a reference to the record variable
Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(
new VariableReferenceExpression(recordVar));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
index 2b2aa90..37e888d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalIndexingOperations.java
@@ -215,7 +215,7 @@ public class ExternalIndexingOperations {
public static JobSpecification buildFilesIndexReplicationJobSpec(Dataset dataset,
ArrayList<ExternalFile> externalFilesSnapshot, AqlMetadataProvider metadataProvider, boolean createIndex)
- throws MetadataException, AlgebricksException {
+ throws MetadataException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
@@ -301,7 +301,7 @@ public class ExternalIndexingOperations {
*/
public static boolean isDatasetUptodate(Dataset dataset, List<ExternalFile> metadataFiles,
List<ExternalFile> addedFiles, List<ExternalFile> deletedFiles, List<ExternalFile> appendedFiles)
- throws MetadataException, AlgebricksException {
+ throws MetadataException, AlgebricksException {
boolean uptodate = true;
int newFileNumber = metadataFiles.get(metadataFiles.size() - 1).getFileNumber() + 1;
@@ -561,7 +561,7 @@ public class ExternalIndexingOperations {
private static ExternalRTreeDataflowHelperFactory getRTreeDataflowHelperFactory(Dataset ds, Index index,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
AsterixStorageProperties storageProperties, AqlMetadataProvider metadataProvider, JobSpecification spec)
- throws AlgebricksException, AsterixException {
+ throws AlgebricksException, AsterixException {
int numPrimaryKeys = getRIDSize(ds);
List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
secondaryKeyFields.size();
@@ -572,6 +572,7 @@ public class ExternalIndexingOperations {
if (spatialType == null) {
throw new AsterixException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
}
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numNestedSecondaryKeyFields = numDimensions * 2;
IPrimitiveValueProviderFactory[] valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
@@ -610,7 +611,7 @@ public class ExternalIndexingOperations {
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true);
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds), true, isPointMBR);
}
public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
index 5512577..6a6538e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -49,8 +49,8 @@ import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConf
import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider;
import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
@@ -96,9 +96,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
secondaryBTreeFields, secondaryFilterFields);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
LocalResource.LSMBTreeResource);
- indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
- dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
- new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(
+ new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), mergePolicyFactory,
+ mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
secondaryBTreeFields, secondaryFilterFields, !dataset.getDatasetDetails().isTemp());
@@ -133,6 +133,7 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+ int[] fieldPermutation = createFieldPermutationForBulkLoadOp(numSecondaryKeys);
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
/*
* In case of external data, this method is used to build loading jobs for both initial load on index creation
@@ -148,12 +149,13 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
sourceOp = createCastOp(spec, primaryScanOp, numSecondaryKeys, dataset.getDatasetType());
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numSecondaryKeys);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numSecondaryKeys,
+ secondaryRecDesc);
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isEnforcingKeyTypes) {
- selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+ selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys, secondaryRecDesc);
}
// Sort by secondary keys.
@@ -163,20 +165,21 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
// Create secondary BTree bulk load op.
AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
ExternalBTreeWithBuddyDataflowHelperFactory dataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(
- mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
IOperatorDescriptor root;
if (externalFiles != null) {
// Transaction load
- secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numSecondaryKeys, dataflowHelperFactory,
+ secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, fieldPermutation, dataflowHelperFactory,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
root = secondaryBulkLoadOp;
} else {
// Initial load
- secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys, dataflowHelperFactory,
+ secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory,
GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
@@ -208,12 +211,13 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
sourceOp = createCastOp(spec, primaryScanOp, numSecondaryKeys, dataset.getDatasetType());
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, sourceOp, numSecondaryKeys);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, sourceOp, numSecondaryKeys,
+ secondaryRecDesc);
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isEnforcingKeyTypes) {
- selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+ selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys, secondaryRecDesc);
}
// Sort by secondary keys.
@@ -222,19 +226,19 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
boolean temp = dataset.getDatasetDetails().isTemp();
// Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
- spec,
- numSecondaryKeys,
+ TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation,
new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
- .getBloomFilterFalsePositiveRate(), false, filterTypeTraits, filterCmpFactories,
- secondaryBTreeFields, secondaryFilterFields, !temp), GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ LSMBTreeIOOperationCallbackFactory.INSTANCE,
+ storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
+ filterCmpFactories, secondaryBTreeFields, secondaryFilterFields, !temp),
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
- new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] { secondaryRecDesc });
+ new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
+ new RecordDescriptor[] { secondaryRecDesc });
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
@@ -268,10 +272,11 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider, secondaryTypeTraits,
- secondaryComparatorFactories, secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(
- new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), mergePolicyFactory,
- mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+ new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE,
storageProperties.getBloomFilterFalsePositiveRate(), false, filterTypeTraits,
filterCmpFactories, secondaryBTreeFields, secondaryFilterFields, !temp),
@@ -285,8 +290,8 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
- .getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
+ LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+ storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
NoOpOperationCallbackFactory.INSTANCE);
}
@@ -363,4 +368,12 @@ public class SecondaryBTreeOperationsHelper extends SecondaryIndexOperationsHelp
enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
}
+
+ protected int[] createFieldPermutationForBulkLoadOp(int numSecondaryKeyFields) {
+ int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys + numFilterFields];
+ for (int i = 0; i < fieldPermutation.length; i++) {
+ fieldPermutation[i] = i;
+ }
+ return fieldPermutation;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index bd49293..2f017e5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -153,7 +153,7 @@ public abstract class SecondaryIndexOperationsHelper {
String datasetName, String indexName, List<List<String>> secondaryKeyFields, List<IAType> secondaryKeyTypes,
boolean isEnforced, int gramLength, AqlMetadataProvider metadataProvider,
PhysicalOptimizationConfig physOptConf, ARecordType recType, ARecordType enforcedType)
- throws AsterixException, AlgebricksException {
+ throws AsterixException, AlgebricksException {
IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
SecondaryIndexOperationsHelper indexOperationsHelper = null;
switch (indexType) {
@@ -346,7 +346,8 @@ public abstract class SecondaryIndexOperationsHelper {
}
protected AlgebricksMetaOperatorDescriptor createAssignOp(JobSpecification spec,
- AbstractOperatorDescriptor primaryScanOp, int numSecondaryKeyFields) throws AlgebricksException {
+ AbstractOperatorDescriptor primaryScanOp, int numSecondaryKeyFields, RecordDescriptor secondaryRecDesc)
+ throws AlgebricksException {
int[] outColumns = new int[numSecondaryKeyFields + numFilterFields];
int[] projectionList = new int[numSecondaryKeyFields + numPrimaryKeys + numFilterFields];
for (int i = 0; i < numSecondaryKeyFields + numFilterFields; i++) {
@@ -419,12 +420,8 @@ public abstract class SecondaryIndexOperationsHelper {
}
protected TreeIndexBulkLoadOperatorDescriptor createTreeIndexBulkLoadOp(JobSpecification spec,
- int numSecondaryKeyFields, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
- throws MetadataException, AlgebricksException {
- int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys + numFilterFields];
- for (int i = 0; i < fieldPermutation.length; i++) {
- fieldPermutation[i] = i;
- }
+ int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
+ throws MetadataException, AlgebricksException {
TreeIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new TreeIndexBulkLoadOperatorDescriptor(spec,
secondaryRecDesc, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
@@ -435,8 +432,8 @@ public abstract class SecondaryIndexOperationsHelper {
return treeIndexBulkLoadOp;
}
- public AlgebricksMetaOperatorDescriptor createFilterNullsSelectOp(JobSpecification spec, int numSecondaryKeyFields)
- throws AlgebricksException {
+ public AlgebricksMetaOperatorDescriptor createFilterNullsSelectOp(JobSpecification spec, int numSecondaryKeyFields,
+ RecordDescriptor secondaryRecDesc) throws AlgebricksException {
IScalarEvaluatorFactory[] andArgsEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeyFields];
NotDescriptor notDesc = new NotDescriptor();
IsNullDescriptor isNullDesc = new IsNullDescriptor();
@@ -500,8 +497,8 @@ public abstract class SecondaryIndexOperationsHelper {
return indexingOpAndConstraints.first;
}
- protected AlgebricksMetaOperatorDescriptor createExternalAssignOp(JobSpecification spec, int numSecondaryKeys)
- throws AlgebricksException {
+ protected AlgebricksMetaOperatorDescriptor createExternalAssignOp(JobSpecification spec, int numSecondaryKeys,
+ RecordDescriptor secondaryRecDesc) throws AlgebricksException {
int[] outColumns = new int[numSecondaryKeys];
int[] projectionList = new int[numSecondaryKeys + numPrimaryKeys];
for (int i = 0; i < numSecondaryKeys; i++) {
@@ -525,12 +522,8 @@ public abstract class SecondaryIndexOperationsHelper {
}
protected ExternalIndexBulkModifyOperatorDescriptor createExternalIndexBulkModifyOp(JobSpecification spec,
- int numSecondaryKeyFields, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
- throws MetadataException, AlgebricksException {
- int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys];
- for (int i = 0; i < numSecondaryKeyFields + numPrimaryKeys; i++) {
- fieldPermutation[i] = i;
- }
+ int[] fieldPermutation, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
+ throws MetadataException, AlgebricksException {
// create a list of file ids
int numOfDeletedFiles = 0;
for (ExternalFile file : externalFiles) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
index 4b9eff7..e072a63 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryInvertedIndexOperationsHelper.java
@@ -253,13 +253,13 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryIndexOperat
sourceOp = createCastOp(spec, primaryScanOp, numSecondaryKeys, dataset.getDatasetType());
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, sourceOp, numSecondaryKeys);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, sourceOp, numSecondaryKeys, secondaryRecDesc);
// If any of the secondary fields are nullable, then add a select op
// that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isEnforcingKeyTypes) {
- selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+ selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys, secondaryRecDesc);
}
// Create a tokenizer op.
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
index be1c356..d3788fc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -86,6 +86,8 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
protected ATypeTag keyType;
protected int[] primaryKeyFields;
protected int[] rtreeFields;
+ protected boolean isPointMBR;
+ protected RecordDescriptor secondaryRecDescForPointMBR = null;
protected SecondaryRTreeOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
@@ -104,39 +106,40 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(secondaryTypeTraits,
secondaryComparatorFactories, primaryComparatorFactories, valueProviderFactories,
- RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
- mergePolicyFactoryProperties, filterTypeTraits, filterCmpFactories, rtreeFields, primaryKeyFields,
- secondaryFilterFields);
+ RTreePolicyType.RTREE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+ dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties, filterTypeTraits,
+ filterCmpFactories, rtreeFields, primaryKeyFields, secondaryFilterFields, isPointMBR);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
LocalResource.LSMRTreeResource);
indexDataflowHelperFactory = new LSMRTreeDataflowHelperFactory(valueProviderFactories,
- RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
- dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
- new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ RTreePolicyType.RTREE, primaryComparatorFactories,
+ new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), mergePolicyFactory,
+ mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp);
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp, isPointMBR);
} else {
// External dataset
// Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new ExternalRTreeLocalResourceMetadata(secondaryTypeTraits,
secondaryComparatorFactories, ExternalIndexingOperations.getBuddyBtreeComparatorFactories(),
- valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
- mergePolicyFactoryProperties, primaryKeyFields);
+ valueProviderFactories, RTreePolicyType.RTREE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+ dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties, primaryKeyFields,
+ isPointMBR);
localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
LocalResource.ExternalRTreeResource);
indexDataflowHelperFactory = new ExternalRTreeDataflowHelperFactory(valueProviderFactories,
RTreePolicyType.RTREE, ExternalIndexingOperations.getBuddyBtreeComparatorFactories(),
- mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
- new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+ storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true, isPointMBR);
}
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
@@ -158,14 +161,12 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
@Override
protected void setSecondaryRecDescAndComparators(IndexType indexType, List<List<String>> secondaryKeyFields,
- List<IAType> secondaryKeyTypes, int gramLength, AqlMetadataProvider metadata) throws AlgebricksException,
- AsterixException {
+ List<IAType> secondaryKeyTypes, int gramLength, AqlMetadataProvider metadata)
+ throws AlgebricksException, AsterixException {
int numSecondaryKeys = secondaryKeyFields.size();
if (numSecondaryKeys != 1) {
- throw new AsterixException(
- "Cannot use "
- + numSecondaryKeys
- + " fields as a key for the R-tree index. There can be only one field as a key for the R-tree index.");
+ throw new AsterixException("Cannot use " + numSecondaryKeys
+ + " fields as a key for the R-tree index. There can be only one field as a key for the R-tree index.");
}
Pair<IAType, Boolean> spatialTypePair = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
secondaryKeyFields.get(0), itemType);
@@ -174,6 +175,7 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
if (spatialType == null) {
throw new AsterixException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
}
+ isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
numNestedSecondaryKeyFields = numDimensions * 2;
int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
@@ -193,8 +195,8 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
ISerializerDeserializer keySerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(nestedKeyType);
secondaryRecFields[i] = keySerde;
- secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(
- nestedKeyType, true);
+ secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE
+ .getBinaryComparatorFactory(nestedKeyType, true);
secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
@@ -234,12 +236,43 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
for (int i = 0; i < primaryKeyFields.length; i++) {
primaryKeyFields[i] = i + numNestedSecondaryKeyFields;
}
+ if (isPointMBR) {
+ int numNestedSecondaryKeyFieldForPointMBR = numNestedSecondaryKeyFields / 2;
+ ISerializerDeserializer[] recFieldsForPointMBR = new ISerializerDeserializer[numPrimaryKeys
+ + numNestedSecondaryKeyFieldForPointMBR + numFilterFields];
+ int idx = 0;
+ for (int i = 0; i < numNestedSecondaryKeyFieldForPointMBR; i++) {
+ recFieldsForPointMBR[idx++] = secondaryRecFields[i];
+ }
+ for (int i = 0; i < numPrimaryKeys + numFilterFields; i++) {
+ recFieldsForPointMBR[idx++] = secondaryRecFields[numNestedSecondaryKeyFields + i];
+ }
+ secondaryRecDescForPointMBR = new RecordDescriptor(recFieldsForPointMBR);
+ }
}
@Override
public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
+ /***************************************************
+ * [ About PointMBR Optimization ]
+ * Instead of storing a MBR(4 doubles) for a point(2 doubles) in RTree leaf node, PointMBR concept is introduced.
+ * PointMBR is a way to store a point as 2 doubles in RTree leaf node.
+ * This reduces RTree index size roughly in half.
+ * In order to fully benefit from the PointMBR concept, besides RTree,
+ * external sort operator during bulk-loading (from either data loading or index creation)
+ * must deal with point as 2 doubles instead of 4 doubles. Otherwise, external sort will suffer from twice as
+ * many doubles as it actually requires. For this purpose, PointMBR specific optimization logic is added as follows:
+ * 1) CreateMBR function in assign operator generates 2 doubles, instead of 4 doubles.
+ * 2) External sort operator sorts points represented with 2 doubles.
+ * 3) Bulk-loading in RTree takes 4 doubles by reading 2 doubles twice and then, do the same work as non-point MBR cases.
+ ***************************************************/
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
boolean temp = dataset.getDatasetDetails().isTemp();
+ int[] fieldPermutation = createFieldPermutationForBulkLoadOp(numNestedSecondaryKeyFields);
+ int numNestedSecondaryKeFieldsConsideringPointMBR = isPointMBR ? numNestedSecondaryKeyFields / 2
+ : numNestedSecondaryKeyFields;
+ RecordDescriptor secondaryRecDescConsideringPointMBR = isPointMBR ? secondaryRecDescForPointMBR
+ : secondaryRecDesc;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
// Create dummy key provider for feeding the primary index scan.
AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
@@ -254,33 +287,33 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
}
AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, sourceOp,
- numNestedSecondaryKeyFields);
+ numNestedSecondaryKeFieldsConsideringPointMBR, secondaryRecDescConsideringPointMBR);
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isEnforcingKeyTypes) {
- selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
+ selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
+ secondaryRecDescConsideringPointMBR);
}
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
- new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length) }, secondaryRecDesc);
+ new IBinaryComparatorFactory[] {
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length) },
+ isPointMBR ? secondaryRecDescForPointMBR : secondaryRecDesc);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary RTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
- spec,
- numNestedSecondaryKeyFields,
+ TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation,
new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), storageProperties
- .getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
+ LSMRTreeIOOperationCallbackFactory.INSTANCE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+ storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp, isPointMBR),
GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() }, new RecordDescriptor[] {});
@@ -311,18 +344,21 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
}
// Assign op.
- AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numNestedSecondaryKeyFields);
+ AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec,
+ numNestedSecondaryKeFieldsConsideringPointMBR, secondaryRecDescConsideringPointMBR);
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable || isEnforcingKeyTypes) {
- selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+ selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeFieldsConsideringPointMBR,
+ secondaryRecDescConsideringPointMBR);
}
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
- new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length) }, secondaryRecDesc);
+ new IBinaryComparatorFactory[] {
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length) },
+ isPointMBR ? secondaryRecDescForPointMBR : secondaryRecDesc);
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create the dataflow helper factory
@@ -332,19 +368,19 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true);
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true, isPointMBR);
// Create secondary RTree bulk load op.
IOperatorDescriptor root;
AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
if (externalFiles != null) {
// Transaction load
- secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numNestedSecondaryKeyFields,
- dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, fieldPermutation, dataflowHelperFactory,
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
root = secondaryBulkLoadOp;
} else {
// Initial load
- secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numNestedSecondaryKeyFields,
- dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory,
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
new RecordDescriptor[] { secondaryRecDesc });
@@ -366,6 +402,52 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
return spec;
}
+ protected int[] createFieldPermutationForBulkLoadOp(int numSecondaryKeyFields) {
+ int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys + numFilterFields];
+ int numSecondaryKeyFieldsForPointMBR = numSecondaryKeyFields / 2;
+ int end = isPointMBR ? numSecondaryKeyFieldsForPointMBR : fieldPermutation.length;
+ for (int i = 0; i < end; i++) {
+ fieldPermutation[i] = i;
+ }
+ if (isPointMBR) {
+ /*******************************************************************************
+ * For example, suppose that 2d point type data is indexed using RTree, there is no
+ * filter fields, and a primary key consists of a single field.
+ * ========== Without PointMBR optimization ==========
+ * If there is no point type optimization, the input operator of RTree's TreeIndexBulkLoadOperator
+ * delivers five variables to the TreeIndexBulkLoadOperator as follows:
+ * [$var1, $var2, $var3, $var4, $var5]
+ * where $var1 ~ $var4 together represent an MBR of a point object.
+ * Since it is a point object, $var1 and $var3 have always identical values. So do $var2 and $var3.
+ * $var5 represents a primary key value.
+ * fieldPermutation variable captures this order correctly by putting values in the array as follows:
+ * [0,1,2,3,4]
+ * =========== With PointMBR optimization ===========
+ * With PointMBR optimization, the input operator of RTree's TreeIndexBulkLoadOperator
+ * delivers 3 variables to the TreeIndexBulkLoadOperator as follows:
+ * [$var1, $var2, $var3]
+ * where $var1 and $var2 together represent an MBR of a point object.
+ * $var3 represents a primary key value.
+ * fieldPermutation variable captures this order correctly by putting values in the array as follows:
+ * [0,1,0,1,2]
+ * This means that bulkloadOp reads the pair of $var1 and $var2 twice in order to provide the same
+ * output just like when there were no PointMBR optimization available.
+ * This adjustment is done in this if clause code.
+ *********************************************************************************/
+ int idx = numSecondaryKeyFieldsForPointMBR;
+ //add the rest of the sk fields for pointMBR
+ for (int i = 0; i < numSecondaryKeyFieldsForPointMBR; i++) {
+ fieldPermutation[idx++] = i;
+ }
+ //add the pk and filter fields
+ end = numSecondaryKeyFieldsForPointMBR + numPrimaryKeys + numFilterFields;
+ for (int i = numSecondaryKeyFieldsForPointMBR; i < end; i++) {
+ fieldPermutation[idx++] = i;
+ }
+ }
+ return fieldPermutation;
+ }
+
@Override
public JobSpecification buildCompactJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = JobSpecificationUtils.createJobSpecification();
@@ -377,15 +459,16 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider, secondaryTypeTraits,
- secondaryComparatorFactories, secondaryBloomFilterKeyFields, new LSMRTreeDataflowHelperFactory(
- valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories,
- new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), mergePolicyFactory,
- mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
- dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length),
+ secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+ new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+ primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+ mergePolicyFactory, mergePolicyFactoryProperties,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ LSMRTreeIOOperationCallbackFactory.INSTANCE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, primaryKeyFields,
- filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp),
+ filterTypeTraits, filterCmpFactories, secondaryFilterFields, !temp, isPointMBR),
NoOpOperationCallbackFactory.INSTANCE);
} else {
// External dataset
@@ -397,10 +480,11 @@ public class SecondaryRTreeOperationsHelper extends SecondaryIndexOperationsHelp
primaryComparatorFactories, mergePolicyFactory, mergePolicyFactoryProperties,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), storageProperties
- .getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true),
+ LSMRTreeIOOperationCallbackFactory.INSTANCE,
+ AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+ storageProperties.getBloomFilterFalsePositiveRate(),
+ new int[] { numNestedSecondaryKeyFields },
+ ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset), true, isPointMBR),
NoOpOperationCallbackFactory.INSTANCE);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 90ebfb7..129b6bc 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -859,6 +859,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
throw new AlgebricksException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
}
int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag());
+ boolean isPointMBR = keyType.getTypeTag() == ATypeTag.POINT || keyType.getTypeTag() == ATypeTag.POINT3D;
int numNestedSecondaryKeyFields = numDimensions * 2;
IPrimitiveValueProviderFactory[] valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
@@ -920,7 +921,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
LSMRTreeIOOperationCallbackFactory.INSTANCE,
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields,
- filterTypeTraits, filterCmpFactories, filterFields, !temp),
+ filterTypeTraits, filterCmpFactories, filterFields, !temp, isPointMBR),
retainInput, retainNull, context.getNullWriterFactory(), searchCallbackFactory,
minFilterFieldIndexes, maxFilterFieldIndexes);
@@ -934,7 +935,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
getStorageProperties().getBloomFilterFalsePositiveRate(),
new int[] { numNestedSecondaryKeyFields },
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp, isPointMBR);
// Create the operator
rtreeSearchOp = new ExternalRTreeSearchOperatorDescriptor(jobSpec, outputRecDesc,
appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
@@ -1957,6 +1958,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
secondaryKeyExprs.get(0), recType);
IAType spatialType = keyPairType.first;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = dimension * 2;
int numPrimaryKeys = primaryKeys.size();
@@ -2046,7 +2049,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields, filterTypeTraits,
- filterCmpFactories, filterFields, !temp);
+ filterCmpFactories, filterFields, !temp, isPointMBR);
IOperatorDescriptor op;
if (bulkload) {
long numElementsHint = getCardinalityPerPartitionHint(dataset);
@@ -2067,7 +2070,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
LSMRTreeIOOperationCallbackFactory.INSTANCE,
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields,
- filterTypeTraits, filterCmpFactories, filterFields, !temp),
+ filterTypeTraits, filterCmpFactories, filterFields, !temp, isPointMBR),
filterFactory, false, indexName, null, modificationCallbackFactory,
NoOpOperationCallbackFactory.INSTANCE);
}
@@ -2738,6 +2741,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
secondaryKeyExprs.get(0), recType);
IAType spatialType = keyPairType.first;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = dimension * 2;
int numPrimaryKeys = primaryKeys.size();
@@ -2854,7 +2859,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
LSMRTreeIOOperationCallbackFactory.INSTANCE,
proposeLinearizer(nestedKeyType.getTypeTag(), comparatorFactories.length),
storageProperties.getBloomFilterFalsePositiveRate(), rtreeFields, btreeFields,
- filterTypeTraits, filterCmpFactories, filterFields, !temp),
+ filterTypeTraits, filterCmpFactories, filterFields, !temp, isPointMBR),
filterFactory, false, indexName, null, modificationCallbackFactory,
NoOpOperationCallbackFactory.INSTANCE, prevFieldPermutation);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
index ed468b9..9ed15d2 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/ExternalRTreeLocalResourceMetadata.java
@@ -46,15 +46,16 @@ public class ExternalRTreeLocalResourceMetadata extends LSMRTreeLocalResourceMet
public ExternalRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
- ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] btreeFields) {
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] btreeFields,
+ boolean isPointMBR) {
super(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
linearizeCmpFactory, datasetID, mergePolicyFactory, mergePolicyProperties, null, null, null,
- btreeFields, null);
+ btreeFields, null, isPointMBR);
}
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
+ int partition, int ioDeviceNum) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
try {
return LSMRTreeUtils.createExternalRTree(file, runtimeContextProvider.getBufferCache(),
@@ -66,7 +67,7 @@ public class ExternalRTreeLocalResourceMetadata extends LSMRTreeLocalResourceMet
runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
runtimeContextProvider.getLSMIOScheduler(),
LSMRTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), linearizeCmpFactory,
- btreeFields, -1, true);
+ btreeFields, -1, true, isPointMBR);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index bae36d4..afe9cd2 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -52,13 +52,14 @@ public class LSMRTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetad
protected final Map<String, String> mergePolicyProperties;
protected final int[] rtreeFields;
protected final int[] btreeFields;
+ protected final boolean isPointMBR;
public LSMRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties,
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] rtreeFields,
- int[] btreeFields, int[] filterFields) {
+ int[] btreeFields, int[] filterFields, boolean isPointMBR) {
super(datasetID, filterTypeTraits, filterCmpFactories, filterFields);
this.typeTraits = typeTraits;
this.rtreeCmpFactories = rtreeCmpFactories;
@@ -70,11 +71,12 @@ public class LSMRTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetad
this.mergePolicyProperties = mergePolicyProperties;
this.rtreeFields = rtreeFields;
this.btreeFields = btreeFields;
+ this.isPointMBR = isPointMBR;
}
@Override
public ILSMIndex createIndexInstance(IAsterixAppRuntimeContextProvider runtimeContextProvider, String filePath,
- int partition, int ioDeviceNum) throws HyracksDataException {
+ int partition, int ioDeviceNum) throws HyracksDataException {
FileReference file = new FileReference(new File(filePath));
List<IVirtualBufferCache> virtualBufferCaches = runtimeContextProvider.getDatasetLifecycleManager()
.getVirtualBufferCaches(datasetID, ioDeviceNum);
@@ -88,7 +90,7 @@ public class LSMRTreeLocalResourceMetadata extends AbstractLSMLocalResourceMetad
runtimeContextProvider.getDatasetLifecycleManager().getDatasetInfo(datasetID)),
runtimeContextProvider.getLSMIOScheduler(),
LSMRTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), linearizeCmpFactory,
- rtreeFields, btreeFields, filterTypeTraits, filterCmpFactories, filterFields, true);
+ rtreeFields, btreeFields, filterTypeTraits, filterCmpFactories, filterFields, true, isPointMBR);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
index 50ab6aa..14032a1 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
@@ -20,10 +20,11 @@
package org.apache.hyracks.algebricks.core.algebra.operators.physical;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.commons.lang3.mutable.Mutable;
-
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -84,6 +85,8 @@ public class IndexBulkloadPOperator extends AbstractPhysicalOperator {
@Override
public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) {
+ //skVarMap is used to remove duplicated variable references for order operator
+ Map<Integer, Object> skVarMap = new HashMap<Integer, Object>();
List<LogicalVariable> scanVariables = new ArrayList<>();
scanVariables.addAll(primaryKeys);
scanVariables.add(new LogicalVariable(-1));
@@ -94,7 +97,10 @@ public class IndexBulkloadPOperator extends AbstractPhysicalOperator {
// Data needs to be sorted based on the [token, number of token, PK]
// OR [token, PK] if the index is not partitioned
for (LogicalVariable skVar : secondaryKeys) {
- orderColumns.add(new OrderColumn(skVar, OrderKind.ASC));
+ if (!skVarMap.containsKey(skVar.getId())) {
+ orderColumns.add(new OrderColumn(skVar, OrderKind.ASC));
+ skVarMap.put(skVar.getId(), null);
+ }
}
for (LogicalVariable pkVar : primaryKeys) {
orderColumns.add(new OrderColumn(pkVar, OrderKind.ASC));
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
index ba49067..c1f2cf4 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeOperatorTestHelper.java
@@ -55,6 +55,6 @@ public class LSMRTreeOperatorTestHelper extends LSMTreeOperatorTestHelper {
virtualBufferCacheProvider, new ConstantMergePolicyFactory(), MERGE_POLICY_PROPERTIES,
ThreadCountingOperationTrackerProvider.INSTANCE, SynchronousSchedulerProvider.INSTANCE,
NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, null,
- btreeFields, null, null, null, true);
+ btreeFields, null, null, null, true, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
index 10f33df..403e6ba 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/am/lsm/rtree/LSMRTreeWithAntiMatterTuplesOperatorTestHelper.java
@@ -54,6 +54,6 @@ public class LSMRTreeWithAntiMatterTuplesOperatorTestHelper extends LSMTreeOpera
btreeComparatorFactories, virtualBufferCacheProvider, new ConstantMergePolicyFactory(),
MERGE_POLICY_PROPERTIES, ThreadCountingOperationTrackerProvider.INSTANCE,
SynchronousSchedulerProvider.INSTANCE, NoOpIOOperationCallback.INSTANCE, linearizerCmpFactory, null,
- null, null, null, true);
+ null, null, null, true, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
index d50b52c..7f7cafd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/AbstractLSMRTreeDataflowHelper.java
@@ -49,6 +49,7 @@ public abstract class AbstractLSMRTreeDataflowHelper extends AbstractLSMIndexDat
protected final RTreePolicyType rtreePolicyType;
protected final ILinearizeComparatorFactory linearizeCmpFactory;
protected final int[] rtreeFields;
+ protected final boolean isPointMBR;
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
List<IVirtualBufferCache> virtualBufferCaches, IBinaryComparatorFactory[] btreeComparatorFactories,
@@ -56,11 +57,11 @@ public abstract class AbstractLSMRTreeDataflowHelper extends AbstractLSMIndexDat
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR) {
this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE,
btreeComparatorFactories, valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits,
- filterCmpFactories, filterFields, durable);
+ filterCmpFactories, filterFields, durable, isPointMBR);
}
public AbstractLSMRTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
@@ -70,7 +71,7 @@ public abstract class AbstractLSMRTreeDataflowHelper extends AbstractLSMIndexDat
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR) {
super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory,
ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable);
this.btreeComparatorFactories = btreeComparatorFactories;
@@ -78,16 +79,17 @@ public abstract class AbstractLSMRTreeDataflowHelper extends AbstractLSMIndexDat
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
this.rtreeFields = rtreeFields;
+ this.isPointMBR = isPointMBR;
}
@Override
public ITreeIndex createIndexInstance() throws HyracksDataException {
AbstractTreeIndexOperatorDescriptor treeOpDesc = (AbstractTreeIndexOperatorDescriptor) opDesc;
- return createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx), opDesc
- .getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
+ return createLSMTree(virtualBufferCaches, file, opDesc.getStorageManager().getBufferCache(ctx),
+ opDesc.getStorageManager().getFileMapProvider(ctx), treeOpDesc.getTreeIndexTypeTraits(),
treeOpDesc.getTreeIndexComparatorFactories(), btreeComparatorFactories,
- opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType,
- linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields);
+ opTrackerFactory.getOperationTracker(ctx), valueProviderFactories, rtreePolicyType, linearizeCmpFactory,
+ rtreeFields, filterTypeTraits, filterCmpFactories, filterFields);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/e964f9b8/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
index 79e082c..3503df5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeDataflowHelper.java
@@ -51,10 +51,11 @@ public class ExternalRTreeDataflowHelper extends LSMRTreeDataflowHelper {
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable,
+ boolean isPointMBR) {
super(opDesc, ctx, partition, null, btreeComparatorFactories, valueProviderFactories, rtreePolicyType,
- mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, null,
- btreeFields, null, null, null, durable);
+ mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields,
+ null, null, null, durable, isPointMBR);
this.version = version;
}
@@ -63,10 +64,11 @@ public class ExternalRTreeDataflowHelper extends LSMRTreeDataflowHelper {
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
- ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable) {
+ ILinearizeComparatorFactory linearizeCmpFactory, int[] btreeFields, int version, boolean durable,
+ boolean isPointMBR) {
super(opDesc, ctx, partition, null, bloomFilterFalsePositiveRate, btreeComparatorFactories,
valueProviderFactories, rtreePolicyType, mergePolicy, opTrackerFactory, ioScheduler,
- ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields, null, null, null, durable);
+ ioOpCallbackFactory, linearizeCmpFactory, null, btreeFields, null, null, null, durable, isPointMBR);
this.version = version;
}
@@ -93,11 +95,11 @@ public class ExternalRTreeDataflowHelper extends LSMRTreeDataflowHelper {
ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields)
throws HyracksDataException {
try {
- return LSMRTreeUtils
- .createExternalRTree(file, diskBufferCache, diskFileMapProvider, typeTraits, rtreeCmpFactories,
- btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate,
- mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory.createIOOperationCallback(),
- linearizeCmpFactory, btreeFields, version, durable);
+ return LSMRTreeUtils.createExternalRTree(file, diskBufferCache, diskFileMapProvider, typeTraits,
+ rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory.createIOOperationCallback(), linearizeCmpFactory, btreeFields, version, durable,
+ isPointMBR);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}