You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/05/27 11:26:50 UTC
[kylin] branch 2.3.x updated (c9f8995 -> 72387b1)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from c9f8995 Minor, remove TarGZUtil.java
new 99936df KYLIN-3345 Use Apache Parent POM 19
new 260391f KYLIN-3363 fix wrong partition condition appended in JDBC Source
new 0245bb8 KYLIN-3348 fix 'missing LastBuildJobID' error
new 72387b1 KYLIN-3352 better filter transform for better seg pruning
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/kylin/cube/CubeInstance.java | 23 ++++
.../java/org/apache/kylin/cube/CubeManager.java | 10 +-
.../java/org/apache/kylin/gridtable/GTUtil.java | 28 +++--
.../org/apache/kylin/cube/CubeManagerTest.java | 3 +
.../kylin/metadata/filter/CompareTupleFilter.java | 1 -
.../kylin/storage/gtrecord/DictGridTableTest.java | 127 +++++++++++++++++----
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 5 +-
examples/test_case_data/localmeta/kylin.properties | 2 +-
pom.xml | 2 +-
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 41 ++++---
11 files changed, 187 insertions(+), 57 deletions(-)
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 04/04: KYLIN-3352 better filter transform for better seg
pruning
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 72387b1cf483f2c63542df679816d4127e142f01
Author: Li Yang <li...@apache.org>
AuthorDate: Fri Apr 27 14:39:21 2018 +0800
KYLIN-3352 better filter transform for better seg pruning
---
.../java/org/apache/kylin/gridtable/GTUtil.java | 28 +++--
.../kylin/metadata/filter/CompareTupleFilter.java | 1 -
.../kylin/storage/gtrecord/DictGridTableTest.java | 127 +++++++++++++++++----
3 files changed, 126 insertions(+), 30 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
index 5c9dfe3..65704d5 100644
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTUtil.java
@@ -33,6 +33,7 @@ import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.FilterOptimizeTransformer;
import org.apache.kylin.metadata.filter.IFilterCodeSystem;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.filter.TupleFilterSerializer;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -154,8 +155,7 @@ public class GTUtil {
// In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
// Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
// will always return FALSE.
- if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
- && !TupleFilter.isEvaluableRecursively(filter)) {
+ if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
return ConstantTupleFilter.TRUE;
}
@@ -181,7 +181,6 @@ public class GTUtil {
return filter;
}
- @SuppressWarnings({ "rawtypes", "unchecked" })
protected TupleFilter encodeConstants(CompareTupleFilter oldCompareFilter) {
// extract ColumnFilter & ConstantFilter
TblColRef externalCol = oldCompareFilter.getColumn();
@@ -249,9 +248,13 @@ public class GTUtil {
}
break;
case LT:
- code = translate(col, firstValue, 1);
+ code = translate(col, firstValue, 0);
if (code == null) {
- result = ConstantTupleFilter.TRUE;
+ code = translate(col, firstValue, -1);
+ if (code == null)
+ result = ConstantTupleFilter.FALSE;
+ else
+ result = newCompareFilter(FilterOperatorEnum.LTE, externalCol, code);
} else {
newCompareFilter.addChild(new ConstantTupleFilter(code));
result = newCompareFilter;
@@ -267,9 +270,13 @@ public class GTUtil {
}
break;
case GT:
- code = translate(col, firstValue, -1);
+ code = translate(col, firstValue, 0);
if (code == null) {
- result = ConstantTupleFilter.TRUE;
+ code = translate(col, firstValue, 1);
+ if (code == null)
+ result = ConstantTupleFilter.FALSE;
+ else
+ result = newCompareFilter(FilterOperatorEnum.GTE, externalCol, code);
} else {
newCompareFilter.addChild(new ConstantTupleFilter(code));
result = newCompareFilter;
@@ -290,6 +297,13 @@ public class GTUtil {
return result;
}
+ private TupleFilter newCompareFilter(FilterOperatorEnum op, TblColRef col, ByteArray code) {
+ CompareTupleFilter r = new CompareTupleFilter(op);
+ r.addChild(new ColumnTupleFilter(col));
+ r.addChild(new ConstantTupleFilter(code));
+ return r;
+ }
+
transient ByteBuffer buf;
protected ByteArray translate(int col, Object value, int roundingFlag) {
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
index 4875217..298477f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/CompareTupleFilter.java
@@ -154,7 +154,6 @@ public class CompareTupleFilter extends TupleFilter implements IOptimizeableTupl
// TODO requires generalize, currently only evaluates COLUMN {op} CONST
@Override
- @SuppressWarnings({ "unchecked", "rawtypes" })
public boolean evaluate(IEvaluatableTuple tuple, IFilterCodeSystem cs) {
// extract tuple value
Object tupleValue = null;
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index e80a67c..073c12c 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -34,7 +34,7 @@ import org.apache.kylin.common.util.ImmutableBitSet;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
-import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.dimension.DictionaryDimEnc;
@@ -441,19 +441,61 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LT, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
List<TblColRef> colMapping = Lists.newArrayList();
colMapping.add(extColA);
colMapping.add(extColB);
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+
+ // $1<"9" round down to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "9"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
- // $1<"9" round up to $1<"10"
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals(
- "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]",
- newFilter.toString());
+ // $1<"10" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "10"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LT [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<"11" round down to <="10"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LT, "11"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<="9" round down to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "9"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
+
+ // $1<="10" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "10"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
+
+ // $1<="11" round down to <="10"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.LTE, "11"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 LTE [\\x00]]",
+ newFilter.toString());
+ }
}
@Test
@@ -464,17 +506,61 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
TblColRef extColA = TblColRef.mockup(extTable, 1, "A", "timestamp");
TblColRef extColB = TblColRef.mockup(extTable, 2, "B", "integer");
- CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
- CompareTupleFilter fComp2 = compare(extColB, FilterOperatorEnum.LTE, "9");
- LogicalTupleFilter filter = and(fComp1, fComp2);
-
List<TblColRef> colMapping = Lists.newArrayList();
colMapping.add(extColA);
colMapping.add(extColB);
+
+ CompareTupleFilter fComp1 = compare(extColA, FilterOperatorEnum.GT, "2015-01-14");
+
+ // $1>"101" round up to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "101"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
- // $1<="9" round down to FALSE
- TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
- assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ // $1>"100" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "100"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GT [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>"99" round up to >="100"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GT, "99"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>="101" round up to FALSE
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "101"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(ConstantTupleFilter.FALSE, newFilter);
+ }
+
+ // $1>="100" needs no rounding
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "100"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
+
+ // $1>="99" round up to >="100"
+ {
+ LogicalTupleFilter filter = and(fComp1, compare(extColB, FilterOperatorEnum.GTE, "99"));
+ TupleFilter newFilter = GTUtil.convertFilterColumnsAndConstants(filter, info, colMapping, null);
+ assertEquals(
+ "AND [UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.0 GT [\\x00\\x00\\x01J\\xE5\\xBD\\x5C\\x00], UNKNOWN_MODEL:NULL.GT_MOCKUP_TABLE.1 GTE [\\x09]]",
+ newFilter.toString());
+ }
}
@Test
@@ -639,7 +725,6 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return info;
}
- @SuppressWarnings("unchecked")
private static CubeCodeSystem newDictCodeSystem() {
DimensionEncoding[] dimEncs = new DimensionEncoding[3];
dimEncs[1] = new DictionaryDimEnc(newDictionaryOfInteger());
@@ -647,7 +732,6 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return new CubeCodeSystem(dimEncs);
}
- @SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfString() {
TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter());
builder.addValue("Dong");
@@ -663,9 +747,8 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
return builder.build(0);
}
- @SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder builder = new NumberDictionaryBuilder();
+ NumberDictionaryForestBuilder builder = new NumberDictionaryForestBuilder();
builder.addValue("10");
builder.addValue("20");
builder.addValue("30");
@@ -676,7 +759,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
builder.addValue("80");
builder.addValue("90");
builder.addValue("100");
- return builder.build(0);
+ return builder.build();
}
public static ImmutableBitSet setOf(int... values) {
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 03/04: KYLIN-3348 fix 'missing LastBuildJobID' error
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 0245bb8dca89de9911b9cd88d6ab19e3ba05039d
Author: Li Yang <li...@apache.org>
AuthorDate: Wed Apr 25 15:28:47 2018 +0800
KYLIN-3348 fix 'missing LastBuildJobID' error
---
.../java/org/apache/kylin/cube/CubeInstance.java | 23 ++++++++++++++++++++++
.../java/org/apache/kylin/cube/CubeManager.java | 10 ++++++----
.../org/apache/kylin/cube/CubeManagerTest.java | 3 +++
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 2 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 5 ++---
examples/test_case_data/localmeta/kylin.properties | 2 +-
6 files changed, 36 insertions(+), 9 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index ea5006e..38bd9e8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -237,6 +237,29 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return getCanonicalName();
}
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (getClass() != obj.getClass())
+ return false;
+ CubeInstance other = (CubeInstance) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ return true;
+ }
+
// ============================================================================
@Override
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 15bb676..850ccc0 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -802,10 +802,12 @@ public class CubeManager implements IRealizationProvider {
return segment;
}
- public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment newSegment) throws IOException {
- // work on copy instead of cached objects
- CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy
- CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
+ public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment newSegCopy) throws IOException {
+ // double check the updating objects are not on cache
+ if (newSegCopy.getCubeInstance().isCachedAndShared())
+ throw new IllegalStateException();
+
+ CubeInstance cubeCopy = getCube(cube.getName()).latestCopyForWrite();
if (StringUtils.isBlank(newSegCopy.getStorageLocationIdentifier()))
throw new IllegalStateException(
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
index 8953868..cad5094 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java
@@ -211,6 +211,9 @@ public class CubeManagerTest extends LocalFileMetadataTestCase {
CubeSegment seg4 = mgr.appendSegment(cube, null, new SegmentRange(3000L, 4000L), m3, m4);
mgr.updateCubeSegStatus(seg4, SegmentStatusEnum.READY);
+ cube = mgr.getCube(cube.getName());
+ assertTrue(cube.getSegments().size() == 4);
+
CubeSegment merge1 = mgr.mergeSegments(cube, null, new SegmentRange(0L, 2000L), true);
merge1.setStatus(SegmentStatusEnum.NEW);
merge1.setLastBuildJobID("test");
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index d085a77..7262383 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -56,7 +56,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index 018abab..2e3e69e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -39,15 +39,14 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
private static final Logger logger = LoggerFactory.getLogger(UpdateCubeInfoAfterMergeStep.class);
- private final CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-
public UpdateCubeInfoAfterMergeStep() {
super();
}
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
- final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
+ final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
+ final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
CubeSegment mergedSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
if (mergedSegment == null) {
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index 257d69c..7bd3655 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -26,7 +26,7 @@ kylin.env=DEV
kylin.storage.hbase.owner-tag=whoami@kylin.apache.org
# List of web servers in use, this enables one web server instance to sync up with other servers.
-#kylin.server.cluster-servers=localhost:7070
+kylin.server.cluster-servers=
### SOURCE ###
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 02/04: KYLIN-3363 fix wrong partition condition appended in
JDBC Source
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 260391f856e52519605e7d200d84610e69548224
Author: lidongsjtu <li...@apache.org>
AuthorDate: Sat May 26 22:44:30 2018 +0800
KYLIN-3363 fix wrong partition condition appended in JDBC Source
---
.../apache/kylin/source/jdbc/JdbcHiveMRInput.java | 41 +++++++++++++---------
1 file changed, 24 insertions(+), 17 deletions(-)
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
index 457c832..bbaaa4a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcHiveMRInput.java
@@ -30,6 +30,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.model.TableExtDesc.ColumnStats;
import org.apache.kylin.metadata.model.TableRef;
@@ -53,7 +54,7 @@ public class JdbcHiveMRInput extends HiveMRInput {
public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
super(flatDesc);
}
-
+
private KylinConfig getConfig() {
return flatDesc.getDataModel().getConfig();
}
@@ -140,20 +141,19 @@ public class JdbcHiveMRInput extends HiveMRInput {
KylinConfig config = getConfig();
PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
String partCol = null;
- String partitionString = null;
if (partitionDesc.isPartitioned()) {
partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
- partitionString = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
- flatDesc.getSegment(), flatDesc.getSegRange());
}
String splitTable;
+ String splitTableAlias;
String splitColumn;
String splitDatabase;
TblColRef splitColRef = determineSplitColumn();
splitTable = splitColRef.getTableRef().getTableName();
- splitColumn = splitColRef.getName();
+ splitTableAlias = splitColRef.getTableAlias();
+ splitColumn = splitColRef.getExpressionInSourceDB();
splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();
//using sqoop to extract data from jdbc source and dump them to hive
@@ -167,22 +167,29 @@ public class JdbcHiveMRInput extends HiveMRInput {
String filedDelimiter = config.getJdbcSourceFieldDelimiter();
int mapperNum = config.getSqoopMapperNum();
- String bquery = String.format("SELECT min(%s), max(%s) FROM %s.%s", splitColumn, splitColumn, splitDatabase,
- splitTable);
- if (partitionString != null) {
- bquery += " WHERE " + partitionString;
+ String bquery = String.format("SELECT min(%s), max(%s) FROM \"%s\".%s as %s", splitColumn, splitColumn,
+ splitDatabase, splitTable, splitTableAlias);
+ if (partitionDesc.isPartitioned()) {
+ SegmentRange segRange = flatDesc.getSegRange();
+ if (segRange != null && !segRange.isInfinite()) {
+ if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
+ && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
+ .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
+ bquery += " WHERE " + partitionDesc.getPartitionConditionBuilder()
+ .buildDateRangeCondition(partitionDesc, flatDesc.getSegment(), segRange);
+ }
+ }
}
//related to "kylin.engine.mr.config-override.mapreduce.job.queuename"
String queueName = getSqoopJobQueueName(config);
- String cmd = String.format(
- "%s/sqoop import -Dorg.apache.sqoop.splitter.allow_text_splitter=true "
- + "-Dmapreduce.job.queuename=%s "
- + "--connect \"%s\" --driver %s --username %s --password %s --query \"%s AND \\$CONDITIONS\" "
- + "--target-dir %s/%s --split-by %s.%s --boundary-query \"%s\" --null-string '' "
- + "--fields-terminated-by '%s' --num-mappers %d",
- sqoopHome, queueName, connectionUrl, driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable,
- splitTable, splitColumn, bquery, filedDelimiter, mapperNum);
+ String cmd = String.format("%s/sqoop import -Dorg.apache.sqoop.splitter.allow_text_splitter=true "
+ + "-Dmapreduce.job.queuename=%s "
+ + "--connect \"%s\" --driver %s --username %s --password %s --query \"%s AND \\$CONDITIONS\" "
+ + "--target-dir %s/%s --split-by %s.%s --boundary-query \"%s\" --null-string '' "
+ + "--fields-terminated-by '%s' --num-mappers %d", sqoopHome, queueName, connectionUrl, driverClass,
+ jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitTable, splitColumn, bquery,
+ filedDelimiter, mapperNum);
logger.debug(String.format("sqoop cmd:%s", cmd));
CmdStep step = new CmdStep();
step.setCmd(cmd);
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.
[kylin] 01/04: KYLIN-3345 Use Apache Parent POM 19
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.3.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 99936df30dbce5f4d2a9e32bf564381d22fc159d
Author: lidongsjtu <li...@apache.org>
AuthorDate: Sat Apr 28 10:25:01 2018 +0800
KYLIN-3345 Use Apache Parent POM 19
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index e089673..6bb2d04 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>16</version>
+ <version>19</version>
<relativePath />
<!-- no parent resolution -->
</parent>
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.