You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/06/01 05:43:12 UTC
[kylin] 02/08: KYLIN-3374 Some improvements for lookup table -
metadata change
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 179eddbdb858626b537d19112cc39630098e5245
Author: Ma,Gang <ga...@ebay.com>
AuthorDate: Thu May 10 12:38:46 2018 +0800
KYLIN-3374 Some improvements for lookup table - metadata change
---
.../kylin/common/persistence/ResourceStore.java | 1 +
.../java/org/apache/kylin/cube/CubeInstance.java | 18 ++
.../java/org/apache/kylin/cube/model/CubeDesc.java | 47 +++++
.../apache/kylin/cube/model/SnapshotTableDesc.java | 75 ++++++++
.../kylin/dict/lookup/ExtTableSnapshotInfo.java | 149 +++++++++++++++
.../dict/lookup/ExtTableSnapshotInfoManager.java | 209 +++++++++++++++++++++
.../apache/kylin/dict/lookup/SnapshotManager.java | 15 ++
.../apache/kylin/dict/lookup/SnapshotTable.java | 25 ++-
8 files changed, 536 insertions(+), 3 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
index bda6cd0..a71db45 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java
@@ -77,6 +77,7 @@ abstract public class ResourceStore {
public static final String BAD_QUERY_RESOURCE_ROOT = "/bad_query";
public static final String DRAFT_RESOURCE_ROOT = "/draft";
public static final String USER_ROOT = "/user";
+ public static final String EXT_SNAPSHOT_RESOURCE_ROOT = "/ext_table_snapshot";
public static final String METASTORE_UUID_TAG = "/UUID";
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index bf966d4..c0f3536 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -124,6 +125,9 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
@JsonProperty("cuboid_last_optimized")
private long cuboidLastOptimized;
+ @JsonProperty("snapshots")
+ private Map<String, String> snapshots = Maps.newHashMap();
+
// cuboid scheduler lazy built
transient private CuboidScheduler cuboidScheduler;
@@ -675,6 +679,20 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
return getDescriptor().getEngineType();
}
+ public Map<String, String> getSnapshots() {
+ if (snapshots == null)
+ snapshots = Maps.newHashMap();
+ return snapshots;
+ }
+
+ public String getSnapshotResPath(String tableName) {
+ return getSnapshots().get(tableName);
+ }
+
+ public void putSnapshotResPath(String table, String snapshotResPath) {
+ getSnapshots().put(table, snapshotResPath);
+ }
+
public static CubeInstance getCopyOf(CubeInstance cubeInstance) {
CubeInstance newCube = new CubeInstance();
newCube.setName(cubeInstance.getName());
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 77b808b..5b4a134 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -203,6 +203,9 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
// Error messages during resolving json metadata
private List<String> errors = new ArrayList<String>();
+ @JsonProperty("snapshot_table_desc_list")
+ private List<SnapshotTableDesc> snapshotTableDescList = Collections.emptyList();
+
private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<>();
private LinkedHashSet<ColumnDesc> allColumnDescs = new LinkedHashSet<>();
private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<>();
@@ -1371,6 +1374,49 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
return null;
}
+ public List<SnapshotTableDesc> getSnapshotTableDescList() {
+ return snapshotTableDescList;
+ }
+
+ public void setSnapshotTableDescList(List<SnapshotTableDesc> snapshotTableDescList) {
+ this.snapshotTableDescList = snapshotTableDescList;
+ }
+
+ public SnapshotTableDesc getSnapshotTableDesc(String tableName) {
+ for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
+ if (snapshotTableDesc.getTableName().equalsIgnoreCase(tableName)) {
+ return snapshotTableDesc;
+ }
+ }
+ return null;
+ }
+
+ public boolean isGlobalSnapshotTable(String tableName) {
+ SnapshotTableDesc desc = getSnapshotTableDesc(tableName);
+ if (desc == null) {
+ return false;
+ }
+ return desc.isGlobal();
+ }
+
+ public boolean isExtSnapshotTable(String tableName) {
+ SnapshotTableDesc desc = getSnapshotTableDesc(tableName);
+ if (desc == null) {
+ return false;
+ }
+ return desc.isExtSnapshotTable();
+ }
+
+ public List<String> getAllExtLookupSnapshotTypes() {
+ List<String> result = Lists.newArrayList();
+ for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
+ if (snapshotTableDesc.isExtSnapshotTable()) {
+ result.add(snapshotTableDesc.getStorageType());
+ }
+ }
+ return result;
+ }
+
/** Get a column which can be used to cluster the source table.
* To reduce memory footprint in base cuboid for global dict */
// TODO handle more than one ultra high cardinality columns use global dict in one cube
@@ -1465,6 +1511,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart());
newCubeDesc.setVersion(cubeDesc.getVersion());
newCubeDesc.setParentForward(cubeDesc.getParentForward());
+ newCubeDesc.setSnapshotTableDescList(cubeDesc.getSnapshotTableDescList());
newCubeDesc.updateRandomUuid();
return newCubeDesc;
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java
new file mode 100644
index 0000000..e61240b
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/SnapshotTableDesc.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.model;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.kylin.dict.lookup.SnapshotTable;
+
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class SnapshotTableDesc implements java.io.Serializable{
+ @JsonProperty("table_name")
+ private String tableName;
+
+ @JsonProperty("storage_type")
+ private String storageType = SnapshotTable.STORAGE_TYPE_METASTORE;
+
+ @JsonProperty("local_cache_enable")
+ private boolean enableLocalCache = true;
+
+ @JsonProperty("global")
+ private boolean global = false;
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getStorageType() {
+ return storageType;
+ }
+
+ public void setStorageType(String storageType) {
+ this.storageType = storageType;
+ }
+
+ public boolean isGlobal() {
+ return global;
+ }
+
+ public void setGlobal(boolean global) {
+ this.global = global;
+ }
+
+ public boolean isExtSnapshotTable() {
+ return !SnapshotTable.STORAGE_TYPE_METASTORE.equals(storageType);
+ }
+
+ public boolean isEnableLocalCache() {
+ return enableLocalCache;
+ }
+
+ public void setEnableLocalCache(boolean enableLocalCache) {
+ this.enableLocalCache = enableLocalCache;
+ }
+}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfo.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfo.java
new file mode 100644
index 0000000..80a2b77
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfo.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict.lookup;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.source.IReadableTable.TableSignature;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@SuppressWarnings("serial")
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
+public class ExtTableSnapshotInfo extends RootPersistentEntity {
+ public static final String STORAGE_TYPE_HBASE = "hbase";
+
+ @JsonProperty("tableName")
+ private String tableName;
+
+ @JsonProperty("signature")
+ private TableSignature signature;
+
+ @JsonProperty("key_columns")
+ private String[] keyColumns;
+
+ @JsonProperty("storage_type")
+ private String storageType;
+
+ @JsonProperty("storage_location_identifier")
+ private String storageLocationIdentifier;
+
+ @JsonProperty("shard_num")
+ private int shardNum;
+
+ @JsonProperty("row_cnt")
+ private long rowCnt;
+
+ @JsonProperty("last_build_time")
+ private long lastBuildTime;
+
+ // default constructor for JSON serialization
+ public ExtTableSnapshotInfo() {
+ }
+
+ public ExtTableSnapshotInfo(TableSignature signature, String tableName) throws IOException {
+ this.signature = signature;
+ this.tableName = tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getResourcePath() {
+ return getResourcePath(tableName, uuid);
+ }
+
+ public String getResourceDir() {
+ return getResourceDir(tableName);
+ }
+
+ public static String getResourcePath(String tableName, String uuid) {
+ return getResourceDir(tableName) + "/" + uuid + ".snapshot";
+ }
+
+ public static String getResourceDir(String tableName) {
+ return ResourceStore.EXT_SNAPSHOT_RESOURCE_ROOT + "/" + tableName;
+ }
+
+ public TableSignature getSignature() {
+ return signature;
+ }
+
+ public String getStorageType() {
+ return storageType;
+ }
+
+ public void setStorageType(String storageType) {
+ this.storageType = storageType;
+ }
+
+ public String getStorageLocationIdentifier() {
+ return storageLocationIdentifier;
+ }
+
+ public void setStorageLocationIdentifier(String storageLocationIdentifier) {
+ this.storageLocationIdentifier = storageLocationIdentifier;
+ }
+
+ public String[] getKeyColumns() {
+ return keyColumns;
+ }
+
+ public void setKeyColumns(String[] keyColumns) {
+ this.keyColumns = keyColumns;
+ }
+
+ public int getShardNum() {
+ return shardNum;
+ }
+
+ public void setShardNum(int shardNum) {
+ this.shardNum = shardNum;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setSignature(TableSignature signature) {
+ this.signature = signature;
+ }
+
+ public long getRowCnt() {
+ return rowCnt;
+ }
+
+ public void setRowCnt(long rowCnt) {
+ this.rowCnt = rowCnt;
+ }
+
+ public long getLastBuildTime() {
+ return lastBuildTime;
+ }
+
+ public void setLastBuildTime(long lastBuildTime) {
+ this.lastBuildTime = lastBuildTime;
+ }
+
+}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
new file mode 100644
index 0000000..1892e57
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/ExtTableSnapshotInfoManager.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict.lookup;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.collect.Sets;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.source.IReadableTable.TableSignature;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+public class ExtTableSnapshotInfoManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(ExtTableSnapshotInfoManager.class);
+ public static Serializer<ExtTableSnapshotInfo> SNAPSHOT_SERIALIZER = new JsonSerializer<>(ExtTableSnapshotInfo.class);
+
+ // static cached instances
+ private static final ConcurrentMap<KylinConfig, ExtTableSnapshotInfoManager> SERVICE_CACHE = new ConcurrentHashMap<KylinConfig, ExtTableSnapshotInfoManager>();
+
+ public static ExtTableSnapshotInfoManager getInstance(KylinConfig config) {
+ ExtTableSnapshotInfoManager r = SERVICE_CACHE.get(config);
+ if (r == null) {
+ synchronized (ExtTableSnapshotInfoManager.class) {
+ r = SERVICE_CACHE.get(config);
+ if (r == null) {
+ r = new ExtTableSnapshotInfoManager(config);
+ SERVICE_CACHE.put(config, r);
+ if (SERVICE_CACHE.size() > 1) {
+ logger.warn("More than one singleton exist");
+ }
+ }
+ }
+ }
+ return r;
+ }
+
+ public static void clearCache() {
+ synchronized (SERVICE_CACHE) {
+ SERVICE_CACHE.clear();
+ }
+ }
+
+ // ============================================================================
+
+ private KylinConfig config;
+ private LoadingCache<String, ExtTableSnapshotInfo> snapshotCache; // resource
+
+ private ExtTableSnapshotInfoManager(KylinConfig config) {
+ this.config = config;
+ this.snapshotCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, ExtTableSnapshotInfo>() {
+ @Override
+ public void onRemoval(RemovalNotification<String, ExtTableSnapshotInfo> notification) {
+ ExtTableSnapshotInfoManager.logger.info("Snapshot with resource path " + notification.getKey()
+ + " is removed due to " + notification.getCause());
+ }
+ }).maximumSize(1000)//
+ .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, ExtTableSnapshotInfo>() {
+ @Override
+ public ExtTableSnapshotInfo load(String key) throws Exception {
+ ExtTableSnapshotInfo snapshot = ExtTableSnapshotInfoManager.this.load(key);
+ return snapshot;
+ }
+ });
+ }
+
+ public boolean hasLatestSnapshot(TableSignature signature, String tableName) throws IOException {
+ ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo(signature, tableName);
+ snapshot.updateRandomUuid();
+ ExtTableSnapshotInfo dupSnapshot = checkDupByInfo(snapshot);
+ if (dupSnapshot != null) {
+ return true;
+ }
+ return false;
+ }
+
+ public ExtTableSnapshotInfo getSnapshot(String snapshotResPath) {
+ try {
+ return snapshotCache.get(snapshotResPath);
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public ExtTableSnapshotInfo getSnapshot(String tableName, String snapshotID) {
+ return getSnapshot(ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID));
+ }
+
+ public List<ExtTableSnapshotInfo> getSnapshots(String tableName) throws IOException {
+ String tableSnapshotsPath = ExtTableSnapshotInfo.getResourceDir(tableName);
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ return store.getAllResources(tableSnapshotsPath, ExtTableSnapshotInfo.class, SNAPSHOT_SERIALIZER);
+ }
+
+ public Set<String> getAllExtSnapshotResPaths() throws IOException {
+ Set<String> result = Sets.newHashSet();
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ Set<String> snapshotTablePaths = store.listResources(ResourceStore.EXT_SNAPSHOT_RESOURCE_ROOT);
+ if (snapshotTablePaths == null) {
+ return result;
+ }
+ for (String snapshotTablePath : snapshotTablePaths) {
+ Set<String> snapshotPaths = store.listResources(snapshotTablePath);
+ if (snapshotPaths != null) {
+ result.addAll(snapshotPaths);
+ }
+ }
+ return result;
+ }
+
+ public void removeSnapshot(String tableName, String snapshotID) throws IOException {
+ String snapshotResPath = ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID);
+ snapshotCache.invalidate(snapshotResPath);
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ store.deleteResource(snapshotResPath);
+ }
+
+ /**
+ * create ext table snapshot
+ * @param signature
+ * @param tableName
+ * @param keyColumns
+ *@param storageType
+ * @param storageLocation @return created snapshot
+ * @throws IOException
+ */
+ public ExtTableSnapshotInfo createSnapshot(TableSignature signature, String tableName, String snapshotID, String[] keyColumns,
+ int shardNum, String storageType, String storageLocation) throws IOException {
+ ExtTableSnapshotInfo snapshot = new ExtTableSnapshotInfo();
+ snapshot.setUuid(snapshotID);
+ snapshot.setSignature(signature);
+ snapshot.setTableName(tableName);
+ snapshot.setKeyColumns(keyColumns);
+ snapshot.setStorageType(storageType);
+ snapshot.setStorageLocationIdentifier(storageLocation);
+ snapshot.setShardNum(shardNum);
+ save(snapshot);
+ return snapshot;
+ }
+
+ public void updateSnapshot(ExtTableSnapshotInfo extTableSnapshot) throws IOException {
+ save(extTableSnapshot);
+ snapshotCache.invalidate(extTableSnapshot.getResourcePath());
+ }
+
+ private ExtTableSnapshotInfo checkDupByInfo(ExtTableSnapshotInfo snapshot) throws IOException {
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ String resourceDir = snapshot.getResourceDir();
+ NavigableSet<String> existings = store.listResources(resourceDir);
+ if (existings == null)
+ return null;
+
+ TableSignature sig = snapshot.getSignature();
+ for (String existing : existings) {
+ ExtTableSnapshotInfo existingSnapshot = load(existing);
+ // direct load from store
+ if (existingSnapshot != null && sig.equals(existingSnapshot.getSignature()))
+ return existingSnapshot;
+ }
+ return null;
+ }
+
+ private ExtTableSnapshotInfo load(String resourcePath) throws IOException {
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ ExtTableSnapshotInfo snapshot = store.getResource(resourcePath, ExtTableSnapshotInfo.class, SNAPSHOT_SERIALIZER);
+
+ return snapshot;
+ }
+
+ private void save(ExtTableSnapshotInfo snapshot) throws IOException {
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ String path = snapshot.getResourcePath();
+ store.putResource(path, snapshot, SNAPSHOT_SERIALIZER);
+ }
+
+}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 5192805..dd90b33 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -19,12 +19,15 @@
package org.apache.kylin.dict.lookup;
import java.io.IOException;
+import java.util.List;
import java.util.NavigableSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.IReadableTable;
import org.apache.kylin.source.IReadableTable.TableSignature;
@@ -94,6 +97,18 @@ public class SnapshotManager {
}
}
+ public List<SnapshotTable> getSnapshots(String tableName, TableSignature sourceTableSignature) throws IOException {
+ List<SnapshotTable> result = Lists.newArrayList();
+ String tableSnapshotsPath = SnapshotTable.getResourceDir(tableName);
+ ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
+ result.addAll(store.getAllResources(tableSnapshotsPath, SnapshotTable.class, SnapshotTableSerializer.INFO_SERIALIZER));
+ if (sourceTableSignature != null) {
+ String oldTableSnapshotsPath = SnapshotTable.getOldResourceDir(sourceTableSignature);
+ result.addAll(store.getAllResources(oldTableSnapshotsPath, SnapshotTable.class, SnapshotTableSerializer.INFO_SERIALIZER));
+ }
+ return result;
+ }
+
public void removeSnapshot(String resourcePath) throws IOException {
ResourceStore store = getStore();
store.deleteResource(resourcePath);
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
index 7bf32b1..6cae22b 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotTable.java
@@ -50,6 +50,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class SnapshotTable extends RootPersistentEntity implements IReadableTable {
+ public static final String STORAGE_TYPE_METASTORE = "metaStore";
@JsonProperty("tableName")
private String tableName;
@@ -57,6 +58,8 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl
private TableSignature signature;
@JsonProperty("useDictionary")
private boolean useDictionary;
+ @JsonProperty("last_build_time")
+ private long lastBuildTime;
private ArrayList<int[]> rowIndices;
private Dictionary<String> dict;
@@ -71,6 +74,18 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl
this.useDictionary = true;
}
+ public long getLastBuildTime() {
+ return lastBuildTime;
+ }
+
+ public void setLastBuildTime(long lastBuildTime) {
+ this.lastBuildTime = lastBuildTime;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
public void takeSnapshot(IReadableTable table, TableDesc tableDesc) throws IOException {
this.signature = table.getSignature();
@@ -121,13 +136,17 @@ public class SnapshotTable extends RootPersistentEntity implements IReadableTabl
public String getResourceDir() {
if (Strings.isNullOrEmpty(tableName)) {
- return getOldResourceDir();
+ return getOldResourceDir(signature);
} else {
- return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + tableName;
+ return getResourceDir(tableName);
}
}
- private String getOldResourceDir() {
+ public static String getResourceDir(String tableName) {
+ return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + tableName;
+ }
+
+ public static String getOldResourceDir(TableSignature signature) {
return ResourceStore.SNAPSHOT_RESOURCE_ROOT + "/" + new File(signature.getPath()).getName();
}
--
To stop receiving notification emails like this one, please contact
shaofengshi@apache.org.