You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/01 05:43:17 UTC
[kylin] 07/08: KYLIN-3375 Some improvements for lookup table -
build change - Fix issue when latest ext snapshot exist
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 20c66d803d2ce0634078d790b441799a7af29991
Author: Ma,Gang <ga...@ebay.com>
AuthorDate: Thu May 24 17:16:47 2018 +0800
KYLIN-3375 Some improvements for lookup table - build change - Fix issue when latest ext snapshot exist
---
.../kylin/dict/lookup/ExtTableSnapshotInfoManager.java | 14 +++++++++-----
.../mr/steps/lookup/UpdateCubeAfterSnapshotStep.java | 11 ++++++++---
.../kylin/storage/hbase/lookup/HBaseLookupMRSteps.java | 5 ++++-
3 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
index 1892e57..f51b020 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
@@ -97,14 +97,18 @@ public class ExtTableSnapshotInfoManager {
});
}
- public boolean hasLatestSnapshot(TableSignature signature, String tableName) throws IOException {
+ /**
+ *
+ * @param signature source table signature
+ * @param tableName
+ * @return latest snapshot info
+ * @throws IOException
+ */
+ public ExtTableSnapshotInfo getLatestSnapshot(TableSignature signature, String tableName) throws IOException {
ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo(signature, tableName);
snapshot.updateRandomUuid();
ExtTableSnapshotInfo dupSnapshot = checkDupByInfo(snapshot);
- if (dupSnapshot != null) {
- return true;
- }
- return false;
+ return dupSnapshot;
}
public ExtTableSnapshotInfo getSnapshot(String snapshotResPath) {
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/UpdateCubeAfterSnapshotStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/UpdateCubeAfterSnapshotStep.java
index 463e3b9..42290b0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/UpdateCubeAfterSnapshotStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/UpdateCubeAfterSnapshotStep.java
@@ -57,13 +57,18 @@ public class UpdateCubeAfterSnapshotStep extends AbstractExecutable {
String contextKey = BatchConstants.LOOKUP_EXT_SNAPSHOT_CONTEXT_PFX + lookupTableName;
String snapshotResPath = job.getExtraInfo(contextKey);
-
+ if (snapshotResPath == null) {
+ logger.info("no snapshot path exist in the context, so no need to update snapshot path");
+ return new ExecuteResult();
+ }
CubeDesc cubeDesc = cube.getDescriptor();
try {
logger.info("update snapshot path to cube metadata");
if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
- LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
- snapshotResPath);
+ if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) {
+ LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
+ snapshotResPath);
+ }
} else {
LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
snapshotResPath);
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMRSteps.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMRSteps.java
index 757f6d0..6101fca 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMRSteps.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/lookup/HBaseLookupMRSteps.java
@@ -83,8 +83,11 @@ public class HBaseLookupMRSteps {
TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject());
IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc);
try {
- if (extTableSnapshotInfoManager.hasLatestSnapshot(sourceTable.getSignature(), tableName)) {
+ ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot(sourceTable.getSignature(), tableName);
+ if (latestSnapshot != null) {
logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName);
+ jobFlow.addExtraInfo(BatchConstants.LOOKUP_EXT_SNAPSHOT_CONTEXT_PFX + latestSnapshot.getTableName(),
+ latestSnapshot.getResourcePath());
return;
}
} catch (IOException ioException) {
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.