You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ak...@apache.org on 2020/09/15 14:00:50 UTC
[carbondata] branch master updated: [CARBONDATA-3983] Fix SI
compatability issue after tupleID format changed
This is an automated email from the ASF dual-hosted git repository.
akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 6f6e11d [CARBONDATA-3983] Fix SI compatability issue after tupleID format changed
6f6e11d is described below
commit 6f6e11d18ed211d86969ab27fc3dc38fd6a9361f
Author: ShreelekhyaG <sh...@yahoo.com>
AuthorDate: Fri Sep 11 15:06:06 2020 +0530
[CARBONDATA-3983] Fix SI compatability issue after tupleID format changed
Why is this PR needed?
Read from maintable having SI returns empty resultset when SI is stored with
old tuple id storage format.
What changes were proposed in this PR?
Checked if blockletPath contains old tuple id format, and convert it to
compatible format.
Does this PR introduce any user interface change?
No
Is any new testcase added?
No, tested in cluster
This closes #3922
---
.../core/constants/CarbonCommonConstants.java | 5 +++++
.../expression/conditional/ImplicitExpression.java | 23 +++++++++++++++++++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index e497c73..d9ed63a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1878,6 +1878,11 @@ public final class CarbonCommonConstants {
public static final String UNDERSCORE = "_";
/**
+ * EQUALS
+ */
+ public static final String EQUALS = "=";
+
+ /**
* POINT
*/
public static final String POINT = ".";
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ImplicitExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ImplicitExpression.java
index 8227f3c..ca58c53 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ImplicitExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ImplicitExpression.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.scan.expression.conditional;
+import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -29,6 +30,7 @@ import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.commons.lang.StringUtils;
@@ -58,14 +60,29 @@ public class ImplicitExpression extends Expression {
}
private void addBlockEntry(String blockletPath) {
- String blockId =
- blockletPath.substring(0, blockletPath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
+ String blockId = blockletPath.substring(0, blockletPath.lastIndexOf(File.separator));
+ // Check if blockId contains old tuple id format, and convert it to compatible format.
+ // Example for non-partition table:
+ // Old tuple id format: 0/0/0-0_batchno0-0-0-1599806689305.snappy
+ // New tuple id format: 0/0-0_0-0-0-1599806689305
+ // Example for partition table:
+ // Old tuple id format: c3=aaa/0-101100000100001_batchno0-0-1-1600093044073.snappy
+ // New tuple id format: c3=aaa/0-101100000100001_0-0-1-1600093044073
+ if (blockId.contains(CarbonTablePath.BATCH_PREFIX)) {
+ blockId = CarbonTablePath.getShortBlockId(blockId);
+ // In case of non-partition table, remove index of part prefix (Part0) from blockId.
+ if (!blockId.substring(0, blockId.indexOf(File.separator))
+ .contains(CarbonCommonConstants.EQUALS)) {
+ blockId = blockId.substring(blockId.indexOf(File.separator) + 1);
+ }
+ }
Set<Integer> blockletIds = blockIdToBlockletIdMapping.get(blockId);
if (null == blockletIds) {
blockletIds = new HashSet<>();
blockIdToBlockletIdMapping.put(blockId, blockletIds);
}
- blockletIds.add(Integer.parseInt(blockletPath.substring(blockId.length() + 1)));
+ blockletIds.add(
+ Integer.parseInt(blockletPath.substring(blockletPath.lastIndexOf(File.separator) + 1)));
}
@Override