You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by aj...@apache.org on 2021/09/01 10:56:13 UTC
[carbondata] branch master updated: [CARBONDATA-4278] Avoid
refetching all indexes to get segment properties
This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 226228f [CARBONDATA-4278] Avoid refetching all indexes to get segment properties
226228f is described below
commit 226228f7474f4083d78b7bd8facb4ec1a41bfd56
Author: Mahesh Raju Somalaraju <ma...@huawei.com>
AuthorDate: Mon Aug 23 19:48:09 2021 +0530
[CARBONDATA-4278] Avoid refetching all indexes to get segment properties
Why is this PR needed?
When block index[BlockIndex] is available then no need to prepare indexes[List[BlockIndex] from available segments and partition locations which might cause delay in query performance.
What changes were proposed in this PR?
Call directly get segment properties if block index[BlockIndex] available.
if (segmentIndices.get(0) instanceof BlockIndex) {
segmentProperties =
segmentPropertiesFetcher.getSegmentPropertiesFromIndex(segmentIndices.get(0));
} else {
segmentProperties =
segmentPropertiesFetcher.getSegmentProperties(segment, partitionLocations);
}
getSegmentPropertiesFromIndex is calling directly block index segment properties.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No. Already index related test cases are present which can cover the added code.
This closes #4209
---
.../org/apache/carbondata/core/index/TableIndex.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/index/TableIndex.java b/core/src/main/java/org/apache/carbondata/core/index/TableIndex.java
index af09606..5bfe442 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/TableIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/TableIndex.java
@@ -48,6 +48,7 @@ import org.apache.carbondata.core.indexstore.BlockletDetailsFetcher;
import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
import org.apache.carbondata.core.indexstore.PartitionSpec;
import org.apache.carbondata.core.indexstore.SegmentPropertiesFetcher;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockIndex;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.IndexSchema;
@@ -207,15 +208,22 @@ public final class TableIndex extends OperationEventListener {
Map<Segment, List<Index>> indexes) throws IOException {
Set<String> missingSISegments = filter.getMissingSISegments();
for (Segment segment : segments) {
+ List<Index> segmentIndices = indexes.get(segment);
if (segment == null ||
- indexes.get(segment) == null || indexes.get(segment).isEmpty()) {
+ segmentIndices == null || segmentIndices.isEmpty()) {
continue;
}
boolean isExternalOrMissingSISegment = segment.isExternalSegment() ||
(missingSISegments != null && missingSISegments.contains(segment.getSegmentNo()));
List<Blocklet> pruneBlocklets = new ArrayList<>();
- SegmentProperties segmentProperties =
- segmentPropertiesFetcher.getSegmentProperties(segment, partitionLocations);
+ SegmentProperties segmentProperties;
+ if (segmentIndices.get(0) instanceof BlockIndex) {
+ segmentProperties =
+ segmentPropertiesFetcher.getSegmentPropertiesFromIndex(segmentIndices.get(0));
+ } else {
+ segmentProperties =
+ segmentPropertiesFetcher.getSegmentProperties(segment, partitionLocations);
+ }
if (filter.isResolvedOnSegment(segmentProperties)) {
FilterExecutor filterExecutor;
if (!isExternalOrMissingSISegment) {
@@ -227,7 +235,7 @@ public final class TableIndex extends OperationEventListener {
.getFilterExecutorTree(filter.getExternalSegmentResolver(), segmentProperties, null,
table.getMinMaxCacheColumns(segmentProperties), false);
}
- for (Index index : indexes.get(segment)) {
+ for (Index index : segmentIndices) {
if (!isExternalOrMissingSISegment) {
pruneBlocklets.addAll(index
.prune(filter.getResolver(), segmentProperties, filterExecutor, this.table));
@@ -249,7 +257,7 @@ public final class TableIndex extends OperationEventListener {
new IndexFilter(segmentProperties, table, expression).getExternalSegmentResolver(),
segmentProperties, null, table.getMinMaxCacheColumns(segmentProperties), false);
}
- for (Index index : indexes.get(segment)) {
+ for (Index index : segmentIndices) {
if (!isExternalOrMissingSISegment) {
pruneBlocklets.addAll(index.prune(
filter.getExpression(), segmentProperties, table, filterExecutor));