You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/10/08 15:04:15 UTC
[01/11] kylin git commit: Sleep 3 seconds before do query pushdown
test in case cache not wiped
Repository: kylin
Updated Branches:
refs/heads/master c04be637d -> 2ca82411c
Sleep 3 seconds before do query pushdown test in case cache not wiped
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/bd7cb9c9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/bd7cb9c9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/bd7cb9c9
Branch: refs/heads/master
Commit: bd7cb9c9570f3d3cb0ceeb8f4c6be81853eb59bd
Parents: ef7d290
Author: nichunen <ch...@kyligence.io>
Authored: Fri Sep 29 19:33:37 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Sep 29 19:37:44 2017 +0800
----------------------------------------------------------------------
build/smoke-test/testQuery.py | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/bd7cb9c9/build/smoke-test/testQuery.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testQuery.py b/build/smoke-test/testQuery.py
index f0eec07..ec1702e 100644
--- a/build/smoke-test/testQuery.py
+++ b/build/smoke-test/testQuery.py
@@ -86,6 +86,9 @@ class testQuery(unittest.TestCase):
self.assertEqual(status_code, 200, 'Disable cube failed.')
+ # Sleep 3 seconds to ensure cache wiped while do query pushdown
+ time.sleep(3)
+
query_url = testQuery.base_url + "/query"
for sql_file in sql_files:
index += 1
[08/11] kylin git commit: KYLIN-2921 Refactor DataModelDesc,
split DataModelManager
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
new file mode 100644
index 0000000..5e32935
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.RawResource;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.model.ExternalFilterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ */
+public class TableMetadataManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(TableMetadataManager.class);
+
+ public static final Serializer<TableDesc> TABLE_SERIALIZER = new JsonSerializer<TableDesc>(TableDesc.class);
+ public static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<TableExtDesc>(
+ TableExtDesc.class);
+ public static final Serializer<ExternalFilterDesc> EXTERNAL_FILTER_DESC_SERIALIZER = new JsonSerializer<ExternalFilterDesc>(
+ ExternalFilterDesc.class);
+
+ // static cached instances
+ private static final ConcurrentMap<KylinConfig, TableMetadataManager> CACHE = new ConcurrentHashMap<KylinConfig, TableMetadataManager>();
+
+ public static TableMetadataManager getInstance(KylinConfig config) {
+ TableMetadataManager r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+
+ synchronized (TableMetadataManager.class) {
+ r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+ try {
+ r = new TableMetadataManager(config);
+ CACHE.put(config, r);
+ if (CACHE.size() > 1) {
+ logger.warn("More than one singleton exist, current keys: {}", StringUtils
+ .join(Iterators.transform(CACHE.keySet().iterator(), new Function<KylinConfig, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable KylinConfig input) {
+ return String.valueOf(System.identityHashCode(input));
+ }
+ }), ","));
+ }
+
+ return r;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to init TableMetadataManager from " + config, e);
+ }
+ }
+ }
+
+ public static void clearCache() {
+ CACHE.clear();
+ }
+
+ // ============================================================================
+
+ private KylinConfig config;
+ // table name ==> SourceTable
+ private CaseInsensitiveStringCache<TableDesc> srcTableMap;
+ // name => SourceTableExt
+ private CaseInsensitiveStringCache<TableExtDesc> srcTableExtMap;
+ // name => External Filter Desc
+ private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap;
+
+ private TableMetadataManager(KylinConfig config) throws IOException {
+ init(config);
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public ResourceStore getStore() {
+ return ResourceStore.getStore(this.config);
+ }
+
+ public List<TableDesc> listAllTables(String prj) {
+ return Lists.newArrayList(getAllTablesMap(prj).values());
+ }
+
+ public List<ExternalFilterDesc> listAllExternalFilters() {
+ return Lists.newArrayList(extFilterMap.values());
+ }
+
+ public Map<String, TableDesc> getAllTablesMap(String prj) {
+ Map<String, TableDesc> globalTables = new LinkedHashMap<>();
+ Map<String, TableDesc> projectTables = new LinkedHashMap<>();
+
+ for (TableDesc t : srcTableMap.values()) {
+ if (t.getProject() == null)
+ globalTables.put(t.getIdentity(), t);
+ else if (t.getProject().equals(prj))
+ projectTables.put(t.getIdentity(), t);
+ }
+
+ Map<String, TableDesc> result = globalTables;
+ result.putAll(projectTables);
+ return result;
+ }
+
+ /**
+ * Get TableDesc by name
+ */
+ public TableDesc getTableDesc(String tableName, String prj) {
+ if (tableName.indexOf(".") < 0)
+ tableName = "DEFAULT." + tableName;
+
+ tableName.toUpperCase();
+
+ TableDesc result = srcTableMap.get(mapKey(tableName, prj));
+ if (result == null)
+ result = srcTableMap.get(mapKey(tableName, null));
+
+ return result;
+ }
+
+ public ExternalFilterDesc getExtFilterDesc(String filterTableName) {
+ ExternalFilterDesc result = extFilterMap.get(filterTableName);
+ return result;
+ }
+
+ /**
+ * Get table extended info. Keys are defined in {@link MetadataConstants}
+ *
+ * @param tableName
+ * @return
+ */
+ public TableExtDesc getTableExt(String tableName, String prj) {
+ TableDesc t = getTableDesc(tableName, prj);
+ if (t == null)
+ return null;
+
+ return getTableExt(t);
+ }
+
+ public TableExtDesc getTableExt(TableDesc t) {
+ TableExtDesc result = srcTableExtMap.get(mapKey(t.getIdentity(), t.getProject()));
+
+ // avoid returning null, since the TableDesc exists
+ if (null == result) {
+ result = new TableExtDesc();
+ result.setIdentity(t.getIdentity());
+ result.setUuid(UUID.randomUUID().toString());
+ result.setLastModified(0);
+ result.init(t.getProject());
+ srcTableExtMap.put(mapKey(t.getIdentity(), t.getProject()), result);
+ }
+ return result;
+ }
+
+ public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException {
+ if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
+ throw new IllegalArgumentException();
+ }
+
+ // updating a legacy global table
+ if (tableExt.getProject() == null) {
+ if (getTableExt(tableExt.getIdentity(), prj).getProject() != null)
+ throw new IllegalStateException(
+ "Updating a legacy global TableExtDesc while a project level version exists: "
+ + tableExt.getIdentity() + ", " + prj);
+ prj = tableExt.getProject();
+ }
+
+ tableExt.init(prj);
+
+ String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj);
+
+ ResourceStore store = getStore();
+
+ TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+ if (t != null && t.getIdentity() == null)
+ store.deleteResource(path);
+
+ store.putResource(path, tableExt, TABLE_EXT_SERIALIZER);
+ srcTableExtMap.put(mapKey(tableExt.getIdentity(), tableExt.getProject()), tableExt);
+ }
+
+ public void removeTableExt(String tableName, String prj) throws IOException {
+ // note, here assume always delete TableExtDesc first, then TableDesc
+ TableExtDesc t = getTableExt(tableName, prj);
+ if (t == null)
+ return;
+
+ String path = TableExtDesc.concatResourcePath(t.getIdentity(), t.getProject());
+ getStore().deleteResource(path);
+ srcTableExtMap.remove(mapKey(t.getIdentity(), t.getProject()));
+ }
+
+ public void saveSourceTable(TableDesc srcTable, String prj) throws IOException {
+ if (srcTable.getUuid() == null || srcTable.getIdentity() == null) {
+ throw new IllegalArgumentException();
+ }
+
+ srcTable.init(prj);
+
+ String path = TableDesc.concatResourcePath(srcTable.getIdentity(), prj);
+ getStore().putResource(path, srcTable, TABLE_SERIALIZER);
+
+ srcTableMap.put(mapKey(srcTable.getIdentity(), prj), srcTable);
+ }
+
+ public void removeSourceTable(String tableIdentity, String prj) throws IOException {
+ TableDesc t = getTableDesc(tableIdentity, prj);
+ if (t == null)
+ return;
+
+ String path = TableDesc.concatResourcePath(t.getIdentity(), t.getProject());
+ getStore().deleteResource(path);
+ srcTableMap.remove(mapKey(t.getIdentity(), t.getProject()));
+ }
+
+ public void saveExternalFilter(ExternalFilterDesc desc) throws IOException {
+ if (desc.getUuid() == null) {
+ throw new IllegalArgumentException("UUID not set.");
+ }
+ String path = desc.getResourcePath();
+ getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER);
+ desc = reloadExternalFilterAt(path);
+ extFilterMap.put(desc.getName(), desc);
+
+ }
+
+ public void removeExternalFilter(String name) throws IOException {
+ String path = ExternalFilterDesc.concatResourcePath(name);
+ getStore().deleteResource(path);
+ extFilterMap.remove(name);
+
+ }
+
+ private void init(KylinConfig config) throws IOException {
+ this.config = config;
+ this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table");
+ this.srcTableExtMap = new CaseInsensitiveStringCache<>(config, "table_ext");
+ this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter");
+
+ reloadAllSourceTable();
+ reloadAllTableExt();
+ reloadAllExternalFilter();
+
+ // touch lower level metadata before registering my listener
+ Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table");
+ Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext");
+ Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter");
+ }
+
+ private class SrcTableSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
+ throws IOException {
+ if (event == Event.DROP)
+ srcTableMap.removeLocal(cacheKey);
+ else
+ reloadSourceTableAt(TableDesc.concatRawResourcePath(cacheKey));
+
+ Pair<String, String> pair = TableDesc.parseResourcePath(cacheKey);
+ String table = pair.getFirst();
+ String prj = pair.getSecond();
+
+ if (prj == null) {
+ for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
+ broadcaster.notifyProjectSchemaUpdate(p.getName());
+ }
+ } else {
+ broadcaster.notifyProjectSchemaUpdate(prj);
+ }
+ }
+ }
+
+ private class SrcTableExtSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
+ throws IOException {
+ if (event == Event.DROP)
+ srcTableExtMap.removeLocal(cacheKey);
+ else
+ reloadTableExtAt(TableExtDesc.concatRawResourcePath(cacheKey));
+ }
+ }
+
+ private class ExtFilterSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
+ throws IOException {
+ if (event == Event.DROP)
+ extFilterMap.removeLocal(cacheKey);
+ else
+ reloadExtFilter(cacheKey);
+ }
+ }
+
+ private void reloadAllTableExt() throws IOException {
+ ResourceStore store = getStore();
+ logger.debug("Reloading Table_exd info from folder "
+ + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT));
+
+ srcTableExtMap.clear();
+
+ List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT,
+ MetadataConstants.FILE_SURFIX);
+ for (String path : paths) {
+ reloadTableExtAt(path);
+ }
+
+ logger.debug("Loaded " + srcTableExtMap.size() + " SourceTable EXD(s)");
+ }
+
+ private TableExtDesc reloadTableExtAt(String path) throws IOException {
+ ResourceStore store = getStore();
+ String prj = TableExtDesc.parseResourcePath(path).getSecond();
+
+ TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+
+ if (t == null) {
+ return null;
+ }
+
+ // convert old tableExt json to new one
+ if (t.getIdentity() == null) {
+ t = convertOldTableExtToNewer(path);
+ }
+
+ t.init(prj);
+
+ srcTableExtMap.putLocal(mapKey(t.getIdentity(), prj), t);
+ return t;
+ }
+
+ private String mapKey(String identity, String prj) {
+ return prj == null ? identity : identity + "--" + prj;
+ }
+
+ private TableExtDesc convertOldTableExtToNewer(String path) throws IOException {
+ Map<String, String> attrs = Maps.newHashMap();
+
+ ResourceStore store = getStore();
+ RawResource res = store.getResource(path);
+
+ InputStream is = res.inputStream;
+
+ try {
+ attrs.putAll(JsonUtil.readValue(is, HashMap.class));
+ } finally {
+ if (is != null)
+ is.close();
+ }
+
+ String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY);
+
+ // parse table identity from file name
+ String file = path;
+ if (file.indexOf("/") > -1) {
+ file = file.substring(file.lastIndexOf("/") + 1);
+ }
+ String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase();
+ TableExtDesc result = new TableExtDesc();
+ result.setIdentity(tableIdentity);
+ result.setUuid(UUID.randomUUID().toString());
+ result.setLastModified(0);
+ result.setCardinality(cardinality);
+ return result;
+ }
+
+ private void reloadAllExternalFilter() throws IOException {
+ ResourceStore store = getStore();
+ logger.debug("Reloading ExternalFilter from folder "
+ + store.getReadableResourcePath(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT));
+
+ extFilterMap.clear();
+
+ List<String> paths = store.collectResourceRecursively(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT,
+ MetadataConstants.FILE_SURFIX);
+ for (String path : paths) {
+ reloadExternalFilterAt(path);
+ }
+
+ logger.debug("Loaded " + extFilterMap.size() + " ExternalFilter(s)");
+ }
+
+ private void reloadAllSourceTable() throws IOException {
+ ResourceStore store = getStore();
+ logger.debug("Reloading SourceTable from folder "
+ + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT));
+
+ srcTableMap.clear();
+
+ List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT,
+ MetadataConstants.FILE_SURFIX);
+ for (String path : paths) {
+ reloadSourceTableAt(path);
+ }
+
+ logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)");
+ }
+
+ private TableDesc reloadSourceTableAt(String path) throws IOException {
+ ResourceStore store = getStore();
+ String prj = TableDesc.parseResourcePath(path).getSecond();
+
+ TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
+ if (t == null) {
+ return null;
+ }
+ t.init(prj);
+
+ srcTableMap.putLocal(mapKey(t.getIdentity(), prj), t);
+
+ return t;
+ }
+
+ private ExternalFilterDesc reloadExternalFilterAt(String path) throws IOException {
+ ResourceStore store = getStore();
+ ExternalFilterDesc t = store.getResource(path, ExternalFilterDesc.class, EXTERNAL_FILTER_DESC_SERIALIZER);
+ if (t == null) {
+ return null;
+ }
+ extFilterMap.putLocal(t.getName(), t);
+
+ return t;
+ }
+
+ public void reloadExtFilter(String extFilterName) throws IOException {
+ reloadExternalFilterAt(ExternalFilterDesc.concatResourcePath(extFilterName));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
index ecb8e61..6d3d541 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/ColumnTupleFilter.java
@@ -26,9 +26,9 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
@@ -134,7 +134,7 @@ public class ColumnTupleFilter extends TupleFilter {
String col = BytesUtil.readUTFString(buffer);
KylinConfig config = KylinConfig.getInstanceFromEnv();
- DataModelDesc modelDesc = MetadataManager.getInstance(config).getDataModelDesc(model);
+ DataModelDesc modelDesc = DataModelManager.getInstance(config).getDataModelDesc(model);
this.columnRef = modelDesc.findColumn(alias, col);
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index e4e311e..edfde43 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -24,7 +24,7 @@ import java.util.Collection;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.filter.ColumnTupleFilter;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
import org.apache.kylin.metadata.filter.FunctionTupleFilter;
@@ -124,7 +124,7 @@ public class MassInTupleFilter extends FunctionTupleFilter {
if (filterTableName == null) {
filterTableName = (String) child.getValues().iterator().next();
- ExternalFilterDesc externalFilterDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getExtFilterDesc(filterTableName);
+ ExternalFilterDesc externalFilterDesc = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getExtFilterDesc(filterTableName);
if (externalFilterDesc == null) {
throw new IllegalArgumentException("External filter named " + filterTableName + " is not found");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 1a48558..f3def90 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -29,15 +29,11 @@ import java.util.Map;
import java.util.Queue;
import java.util.Set;
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringUtil;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.JoinsTree.Chain;
@@ -50,11 +46,6 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -112,10 +103,6 @@ public class DataModelDesc extends RootPersistentEntity {
@JsonProperty("capacity")
private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
- @JsonProperty("computed_columns")
- @JsonInclude(JsonInclude.Include.NON_DEFAULT)
- private List<ComputedColumnDesc> computedColumnDescs = Lists.newArrayList();
-
// computed attributes
private TableRef rootFactTableRef;
private Set<TableRef> factTableRefs = Sets.newLinkedHashSet();
@@ -346,16 +333,7 @@ public class DataModelDesc extends RootPersistentEntity {
throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name);
}
- public void init(KylinConfig config, Map<String, TableDesc> originalTables, List<DataModelDesc> dataModelDescs) {
- //tweak the tables according to Computed Columns defined in model
- Map<String, TableDesc> tables = Maps.newHashMap();
- for (Map.Entry<String, TableDesc> entry : originalTables.entrySet()) {
- String s = entry.getKey();
- TableDesc tableDesc = entry.getValue();
- TableDesc extendedTableDesc = tableDesc.appendColumns(createComputedColumns(tableDesc));
- tables.put(s, extendedTableDesc);
- }
-
+ public void init(KylinConfig config, Map<String, TableDesc> tables, List<DataModelDesc> otherModels) {
this.config = config;
initJoinTablesForUpgrade();
@@ -365,34 +343,14 @@ public class DataModelDesc extends RootPersistentEntity {
initJoinsTree();
initDimensionsAndMetrics();
initPartitionDesc();
- initComputedColumns(dataModelDescs);
initFilterCondition();
boolean reinit = validate();
if (reinit) { // model slightly changed by validate() and must init() again
- init(config, tables, dataModelDescs);
+ init(config, tables, otherModels);
}
}
- private ColumnDesc[] createComputedColumns(final TableDesc tableDesc) {
- final MutableInt id = new MutableInt(tableDesc.getColumnCount());
- return FluentIterable.from(this.computedColumnDescs).filter(new Predicate<ComputedColumnDesc>() {
- @Override
- public boolean apply(@Nullable ComputedColumnDesc input) {
- return tableDesc.getIdentity().equalsIgnoreCase(input.getTableIdentity());
- }
- }).transform(new Function<ComputedColumnDesc, ColumnDesc>() {
- @Nullable
- @Override
- public ColumnDesc apply(@Nullable ComputedColumnDesc input) {
- id.increment();
- ColumnDesc columnDesc = new ColumnDesc(id.toString(), input.getColumnName(), input.getDatatype(),
- input.getComment(), null, null, input.getExpression());
- return columnDesc;
- }
- }).toArray(ColumnDesc.class);
- }
-
private void initJoinTablesForUpgrade() {
if (joinTables == null) {
joinTables = new JoinTableDesc[0];
@@ -486,47 +444,6 @@ public class DataModelDesc extends RootPersistentEntity {
this.partitionDesc.init(this);
}
- private void initComputedColumns(List<DataModelDesc> allDataModelDescs) {
- Preconditions.checkNotNull(allDataModelDescs);
-
- List<Pair<ComputedColumnDesc, DataModelDesc>> existingCCs = Lists.newArrayList();
-
- for (DataModelDesc dataModelDesc : allDataModelDescs) {
- if (!StringUtils.equals(dataModelDesc.getName(), this.getName())) {
- for (ComputedColumnDesc cc : dataModelDesc.getComputedColumnDescs()) {
- existingCCs.add(Pair.newPair(cc, dataModelDesc));
- }
- }
- }
-
- for (ComputedColumnDesc newCC : this.computedColumnDescs) {
-
- newCC.init(aliasMap, rootFactTableRef.getAlias());
- final String newCCFullName = newCC.getFullName();
- final String newCCColumnName = newCC.getColumnName();
-
- for (Pair<ComputedColumnDesc, DataModelDesc> pair : existingCCs) {
- DataModelDesc dataModelDesc = pair.getSecond();
- ComputedColumnDesc cc = pair.getFirst();
-
- if (StringUtils.equalsIgnoreCase(cc.getFullName(), newCCFullName) && !(cc.equals(newCC))) {
- throw new IllegalArgumentException(String.format(
- "Column name for computed column %s is already used in model %s, you should apply the same expression ' %s ' here, or use a different column name.",
- newCCFullName, dataModelDesc.getName(), cc.getExpression()));
- }
-
- if (isTwoCCDefinitionEquals(cc.getExpression(), newCC.getExpression())
- && !StringUtils.equalsIgnoreCase(cc.getColumnName(), newCCColumnName)) {
- throw new IllegalArgumentException(String.format(
- "Expression %s in computed column %s is already defined by computed column %s from model %s, you should use the same column name: ' %s ' .",
- newCC.getExpression(), newCCFullName, cc.getFullName(), dataModelDesc.getName(),
- cc.getColumnName()));
- }
- }
- existingCCs.add(Pair.newPair(newCC, this));
- }
- }
-
//Check if the filter condition is illegal.
private void initFilterCondition() {
if (null == this.filterCondition) {
@@ -569,12 +486,6 @@ public class DataModelDesc extends RootPersistentEntity {
}
}
- private boolean isTwoCCDefinitionEquals(String definition0, String definition1) {
- definition0 = definition0.replaceAll("\\s*", "");
- definition1 = definition1.replaceAll("\\s*", "");
- return definition0.equalsIgnoreCase(definition1);
- }
-
private void initJoinColumns() {
for (JoinTableDesc joinTable : joinTables) {
@@ -813,32 +724,6 @@ public class DataModelDesc extends RootPersistentEntity {
return dimensions;
}
- public ComputedColumnDesc findCCByCCColumnName(final String columnName) {
- return Iterables.find(this.computedColumnDescs, new Predicate<ComputedColumnDesc>() {
- @Override
- public boolean apply(@Nullable ComputedColumnDesc input) {
- Preconditions.checkNotNull(input);
- return columnName.equals(input.getColumnName());
- }
- });
- }
-
- public Set<String> getComputedColumnNames() {
- Set<String> ccColumnNames = Sets.newHashSet();
- for (ComputedColumnDesc cc : this.getComputedColumnDescs()) {
- ccColumnNames.add(cc.getColumnName());
- }
- return Collections.unmodifiableSet(ccColumnNames);
- }
-
- public List<ComputedColumnDesc> getComputedColumnDescs() {
- return computedColumnDescs;
- }
-
- public void setComputedColumnDescs(List<ComputedColumnDesc> computedColumnDescs) {
- this.computedColumnDescs = computedColumnDescs;
- }
-
public String[] getMetrics() {
return metrics;
}
@@ -860,7 +745,10 @@ public class DataModelDesc extends RootPersistentEntity {
}
public static DataModelDesc getCopyOf(DataModelDesc orig) {
- DataModelDesc copy = new DataModelDesc();
+ return copy(orig, new DataModelDesc());
+ }
+
+ public static DataModelDesc copy(DataModelDesc orig, DataModelDesc copy) {
copy.config = orig.config;
copy.name = orig.name;
copy.isDraft = orig.isDraft;
@@ -872,7 +760,6 @@ public class DataModelDesc extends RootPersistentEntity {
copy.metrics = orig.metrics;
copy.filterCondition = orig.filterCondition;
copy.capacity = orig.capacity;
- copy.computedColumnDescs = orig.computedColumnDescs;
if (orig.getPartitionDesc() != null) {
copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
new file mode 100644
index 0000000..fcf47ab
--- /dev/null
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.metadata.MetadataConstants;
+import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
+import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+
+/**
+ */
+public class DataModelManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(DataModelManager.class);
+
+ // static cached instances
+ private static final ConcurrentMap<KylinConfig, DataModelManager> CACHE = new ConcurrentHashMap<KylinConfig, DataModelManager>();
+
+ public static DataModelManager getInstance(KylinConfig config) {
+ DataModelManager r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+
+ synchronized (DataModelManager.class) {
+ r = CACHE.get(config);
+ if (r != null) {
+ return r;
+ }
+ r = newInstance(config);
+ CACHE.put(config, r);
+ if (CACHE.size() > 1) {
+ logger.warn("More than one singleton exist, current keys: {}", StringUtils
+ .join(Iterators.transform(CACHE.keySet().iterator(), new Function<KylinConfig, String>() {
+ @Nullable
+ @Override
+ public String apply(@Nullable KylinConfig input) {
+ return String.valueOf(System.identityHashCode(input));
+ }
+ }), ","));
+ }
+
+ return r;
+ }
+ }
+
+ private static DataModelManager newInstance(KylinConfig conf) {
+ try {
+ String cls = StringUtil.noBlank(conf.getDataModelManagerImpl(), DataModelManager.class.getName());
+ Class<? extends DataModelManager> clz = ClassUtil.forName(cls, DataModelManager.class);
+ return clz.getConstructor(KylinConfig.class).newInstance(conf);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to init DataModelManager from " + conf, e);
+ }
+ }
+
+ public static void clearCache() {
+ CACHE.clear();
+ }
+
+ // ============================================================================
+
+ private KylinConfig config;
+ private Serializer<DataModelDesc> serializer;
+
+ // name => DataModelDesc
+ private CaseInsensitiveStringCache<DataModelDesc> dataModelDescMap;
+
+ public DataModelManager(KylinConfig config) throws IOException {
+ init(config);
+ }
+
+ public KylinConfig getConfig() {
+ return config;
+ }
+
+ public ResourceStore getStore() {
+ return ResourceStore.getStore(this.config);
+ }
+
+ public Serializer<DataModelDesc> getDataModelSerializer() {
+ if (serializer == null) {
+ try {
+ String cls = StringUtil.noBlank(config.getDataModelImpl(), DataModelDesc.class.getName());
+ Class<? extends DataModelDesc> clz = ClassUtil.forName(cls, DataModelDesc.class);
+ serializer = new JsonSerializer(clz);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return serializer;
+ }
+
+ public List<DataModelDesc> listDataModels() {
+ return Lists.newArrayList(this.dataModelDescMap.values());
+ }
+
+ protected void init(KylinConfig config) throws IOException {
+ this.config = config;
+ this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model");
+
+ // touch lower level metadata before registering model listener
+ TableMetadataManager.getInstance(config);
+
+ reloadAllDataModel();
+ Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model");
+ }
+
+ private class DataModelSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ clearCache();
+ }
+
+ @Override
+ public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
+ for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
+ reloadDataModelDescAt(DataModelDesc.concatResourcePath(model));
+ }
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
+ throws IOException {
+ if (event == Event.DROP)
+ dataModelDescMap.removeLocal(cacheKey);
+ else
+ reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey));
+
+ for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) {
+ broadcaster.notifyProjectSchemaUpdate(prj.getName());
+ }
+ }
+ }
+
+ public DataModelDesc getDataModelDesc(String name) {
+ return dataModelDescMap.get(name);
+ }
+
+ public List<DataModelDesc> getModels() {
+ return new ArrayList<>(dataModelDescMap.values());
+ }
+
+ public List<DataModelDesc> getModels(String projectName) {
+ ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
+ ArrayList<DataModelDesc> ret = new ArrayList<>();
+
+ if (projectInstance != null && projectInstance.getModels() != null) {
+ for (String modelName : projectInstance.getModels()) {
+ DataModelDesc model = getDataModelDesc(modelName);
+ if (null != model) {
+ ret.add(model);
+ } else {
+ logger.error("Failed to load model " + modelName);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ // within a project, find models that use the specified table
+ public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException {
+ List<String> models = new ArrayList<>();
+ for (DataModelDesc modelDesc : getModels(project)) {
+ if (modelDesc.containsTable(table))
+ models.add(modelDesc.getName());
+ }
+ return models;
+ }
+
+ public boolean isTableInAnyModel(TableDesc table) {
+ for (DataModelDesc modelDesc : getModels()) {
+ if (modelDesc.containsTable(table))
+ return true;
+ }
+ return false;
+ }
+
+ private void reloadAllDataModel() throws IOException {
+ ResourceStore store = getStore();
+ logger.debug("Reloading DataModel from folder "
+ + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT));
+
+ dataModelDescMap.clear();
+
+ List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT,
+ MetadataConstants.FILE_SURFIX);
+ for (String path : paths) {
+
+ try {
+ logger.info("Reloading data model at " + path);
+ reloadDataModelDescAt(path);
+ } catch (IllegalStateException e) {
+ logger.error("Error to load DataModel at " + path, e);
+ continue;
+ }
+ }
+
+ logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)");
+ }
+
+ public DataModelDesc reloadDataModelDescAt(String path) {
+ ResourceStore store = getStore();
+ try {
+ DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, getDataModelSerializer());
+ String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName();
+
+ if (!dataModelDesc.isDraft()) {
+ dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels());
+ }
+
+ dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
+ return dataModelDesc;
+ } catch (Exception e) {
+ throw new IllegalStateException("Error to load " + path, e);
+ }
+ }
+
+ // sync on update
+ public DataModelDesc dropModel(DataModelDesc desc) throws IOException {
+ logger.info("Dropping model '" + desc.getName() + "'");
+ ResourceStore store = getStore();
+ store.deleteResource(desc.getResourcePath());
+ // delete model from project
+ ProjectManager.getInstance(config).removeModelFromProjects(desc.getName());
+ // clean model cache
+ this.afterModelDropped(desc);
+ return desc;
+ }
+
+ private void afterModelDropped(DataModelDesc desc) {
+ dataModelDescMap.remove(desc.getName());
+ }
+
+ public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException {
+ String name = desc.getName();
+ if (dataModelDescMap.containsKey(name))
+ throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists");
+
+ ProjectManager prjMgr = ProjectManager.getInstance(config);
+ ProjectInstance prj = prjMgr.getProject(projectName);
+ if (prj.containsModel(name))
+ throw new IllegalStateException("project " + projectName + " already contains model " + name);
+
+ try {
+ // Temporarily register model under project, because we want to
+ // update project formally after model is saved.
+ prj.getModels().add(name);
+
+ desc.setOwner(owner);
+ desc = saveDataModelDesc(desc);
+
+ } finally {
+ prj.getModels().remove(name);
+ }
+
+ // now that model is saved, update project formally
+ prjMgr.updateModelToProject(name, projectName);
+
+ return desc;
+ }
+
+ public DataModelDesc updateDataModelDesc(DataModelDesc desc) throws IOException {
+ String name = desc.getName();
+ if (!dataModelDescMap.containsKey(name)) {
+ throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist.");
+ }
+
+ return saveDataModelDesc(desc);
+ }
+
+ private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
+
+ String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName();
+
+ if (!dataModelDesc.isDraft())
+ dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels());
+
+ String path = dataModelDesc.getResourcePath();
+ getStore().putResource(path, dataModelDesc, getDataModelSerializer());
+ dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc);
+
+ return dataModelDesc;
+ }
+
+ private Map<String, TableDesc> getAllTablesMap(String prj) {
+ return TableMetadataManager.getInstance(config).getAllTablesMap(prj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
index 1f54416..5f0e6a3 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectInstance.java
@@ -92,13 +92,6 @@ public class ProjectInstance extends RootPersistentEntity {
return ResourceStore.PROJECT_RESOURCE_ROOT + "/" + projectName + ".json";
}
- public static String getNormalizedProjectName(String project) {
- if (project == null)
- throw new IllegalStateException("Trying to normalized a project name which is null");
-
- return project.toUpperCase();
- }
-
public static ProjectInstance create(String name, String owner, String description, LinkedHashMap<String, String> overrideProps, List<RealizationEntry> realizationEntries, List<String> models) {
ProjectInstance projectInstance = new ProjectInstance();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 1dad89c..df086ec 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -22,8 +22,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
@@ -52,7 +53,7 @@ class ProjectL2Cache {
private static final Logger logger = LoggerFactory.getLogger(ProjectL2Cache.class);
private ProjectManager mgr;
- private Map<String, ProjectCache> projectCaches = Maps.newConcurrentMap();
+ private Map<String, ProjectCache> projectCaches = new ConcurrentSkipListMap<>(String.CASE_INSENSITIVE_ORDER);
ProjectL2Cache(ProjectManager mgr) {
this.mgr = mgr;
@@ -178,7 +179,6 @@ class ProjectL2Cache {
// ----------------------------------------------------------------------------
private ProjectCache getCache(String project) {
- project = ProjectInstance.getNormalizedProjectName(project);
ProjectCache result = projectCaches.get(project);
if (result == null) {
result = loadCache(project);
@@ -196,7 +196,7 @@ class ProjectL2Cache {
if (pi == null)
throw new IllegalArgumentException("Project '" + project + "' does not exist;");
- MetadataManager metaMgr = mgr.getMetadataManager();
+ TableMetadataManager metaMgr = mgr.getTableManager();
for (String tableName : pi.getTables()) {
TableDesc tableDesc = metaMgr.getTableDesc(tableName, project);
@@ -248,7 +248,7 @@ class ProjectL2Cache {
if (realization == null)
return false;
- MetadataManager metaMgr = mgr.getMetadataManager();
+ TableMetadataManager metaMgr = mgr.getTableManager();
Set<TblColRef> allColumns = realization.getAllColumns();
if (allColumns == null || allColumns.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 527233e..8044797 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -34,7 +34,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
@@ -312,16 +312,15 @@ public class ProjectManager {
}
}
- private ProjectInstance addModelToProject(String modelName, String project) throws IOException {
- String newProjectName = ProjectInstance.getNormalizedProjectName(project);
- ProjectInstance newProject = getProject(newProjectName);
- if (newProject == null) {
- throw new IllegalArgumentException("Project " + newProjectName + " does not exist.");
+ private ProjectInstance addModelToProject(String modelName, String prjName) throws IOException {
+ ProjectInstance prj = getProject(prjName);
+ if (prj == null) {
+ throw new IllegalArgumentException("Project " + prjName + " does not exist.");
}
- newProject.addModel(modelName);
- updateProject(newProject);
+ prj.addModel(modelName);
+ updateProject(prj);
- return newProject;
+ return prj;
}
public ProjectInstance moveRealizationToProject(RealizationType type, String realizationName, String newProjectName,
@@ -357,7 +356,7 @@ public class ProjectManager {
}
public ProjectInstance addTableDescToProject(String[] tableIdentities, String projectName) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
ProjectInstance projectInstance = getProject(projectName);
for (String tableId : tableIdentities) {
TableDesc table = metaMgr.getTableDesc(tableId, projectName);
@@ -372,7 +371,7 @@ public class ProjectManager {
}
public void removeTableDescFromProject(String tableIdentities, String projectName) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
ProjectInstance projectInstance = getProject(projectName);
TableDesc table = metaMgr.getTableDesc(tableIdentities, projectName);
if (table == null) {
@@ -384,7 +383,7 @@ public class ProjectManager {
}
public ProjectInstance addExtFilterToProject(String[] filters, String projectName) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
ProjectInstance projectInstance = getProject(projectName);
for (String filterName : filters) {
ExternalFilterDesc extFilter = metaMgr.getExtFilterDesc(filterName);
@@ -399,7 +398,7 @@ public class ProjectManager {
}
public void removeExtFilterFromProject(String filterName, String projectName) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
ProjectInstance projectInstance = getProject(projectName);
ExternalFilterDesc filter = metaMgr.getExtFilterDesc(filterName);
if (filter == null) {
@@ -527,8 +526,8 @@ public class ProjectManager {
return ResourceStore.getStore(this.config);
}
- MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
+ TableMetadataManager getTableManager() {
+ return TableMetadataManager.getInstance(config);
}
private String norm(String project) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
index 8313fce..a8c7303 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ColumnGenConfig.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
public class ColumnGenConfig {
@@ -103,7 +103,7 @@ public class ColumnGenConfig {
KylinConfig kylinConfig = modelGen.getModle().getConfig();
String project = modelGen.getModle().getProject();
- ColumnDesc pkcol = MetadataManager.getInstance(kylinConfig)//
+ ColumnDesc pkcol = TableMetadataManager.getInstance(kylinConfig)//
.getTableDesc(pkTableName, project).findColumnByName(pkColName);
return modelGen.getPkValues(pkcol);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
index 6eb0e71..b7e8d3a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
+++ b/core-metadata/src/main/java/org/apache/kylin/source/datagen/ModelDataGenerator.java
@@ -38,12 +38,12 @@ import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
@@ -307,7 +307,7 @@ public class ModelDataGenerator {
String outputDir = args.length > 2 ? args[2] : null;
KylinConfig conf = KylinConfig.getInstanceFromEnv();
- DataModelDesc model = MetadataManager.getInstance(conf).getDataModelDesc(modelName);
+ DataModelDesc model = DataModelManager.getInstance(conf).getDataModelDesc(modelName);
ResourceStore store = outputDir == null ? ResourceStore.getStore(conf) : ResourceStore.getStore(mockup(outputDir));
ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
deleted file mode 100644
index cfa2324..0000000
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata;
-
-import static org.apache.kylin.metadata.MetadataManager.getInstance;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.ModelDimensionDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableExtDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- */
-public class MetadataManagerTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testCiModel() {
- MetadataManager mgr = getInstance(getTestConfig());
- DataModelDesc lm = mgr.getDataModelDesc("ci_left_join_model");
- DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
- assertSnowflakeQuality(lm);
- assertSnowflakeQuality(im);
-
- // check inner/left models are identical apart from the left/inner difference
- assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
- for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
- JoinTableDesc lt = lm.getJoinTables()[i];
- JoinTableDesc it = im.getJoinTables()[i];
- assertEquals(lt.getAlias(), it.getAlias());
- assertEquals(lt.getKind(), it.getKind());
- assertEquals(lt.getTable(), it.getTable());
- assertArrayEquals(lt.getJoin().getForeignKey(), it.getJoin().getForeignKey());
- assertArrayEquals(lt.getJoin().getPrimaryKey(), it.getJoin().getPrimaryKey());
- assertTrue(lt.getJoin().isLeftJoin());
- assertTrue(it.getJoin().isInnerJoin());
- }
-
- assertEquals(lm.getDimensions().size(), im.getDimensions().size());
- for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
- ModelDimensionDesc ld = lm.getDimensions().get(i);
- ModelDimensionDesc id = im.getDimensions().get(i);
- assertEquals(ld.getTable(), id.getTable());
- assertArrayEquals(ld.getColumns(), id.getColumns());
- }
-
- assertArrayEquals(lm.getMetrics(), im.getMetrics());
- }
-
- private void assertSnowflakeQuality(DataModelDesc model) {
- Assert.assertNotNull(model);
- try {
- model.findTable("TEST_COUNTRY");
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- // excepted
- }
-
- Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
- Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
- Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
- Assert.assertNotNull(model.findColumn("BUYER_ID"));
-
- }
-
- @Test
- public void testListAllTables() throws Exception {
- List<TableDesc> tables = getInstance(getTestConfig()).listAllTables(null);
- Assert.assertNotNull(tables);
- Assert.assertTrue(tables.size() > 0);
- }
-
- @Test
- public void testFindTableByName() throws Exception {
- TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT", "default");
- Assert.assertNotNull(table);
- Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity());
- }
-
- @Test
- public void testGetInstance() throws Exception {
- Assert.assertNotNull(getInstance(getTestConfig()));
- Assert.assertNotNull(getInstance(getTestConfig()).listAllTables(null));
- Assert.assertTrue(getInstance(getTestConfig()).listAllTables(null).size() > 0);
- }
-
- @Test
- public void testDataModel() throws Exception {
- DataModelDesc modelDesc = getInstance(getTestConfig()).getDataModelDesc("test_kylin_left_join_model_desc");
- Assert.assertTrue(modelDesc.getDimensions().size() > 0);
- }
-
- @Test
- public void testTableSample() throws IOException {
- TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default");
- Assert.assertNotNull(tableExtDesc);
-
- List<TableExtDesc.ColumnStats> columnStatsList = new ArrayList<>();
- TableExtDesc.ColumnStats columnStats = new TableExtDesc.ColumnStats();
- columnStats.setColumnSamples("Max", "Min", "dfadsfdsfdsafds", "d");
- columnStatsList.add(columnStats);
- tableExtDesc.setColumnStats(columnStatsList);
- getInstance(getTestConfig()).saveTableExt(tableExtDesc, "default");
-
- TableExtDesc tableExtDesc1 = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default");
- Assert.assertNotNull(tableExtDesc1);
-
- List<TableExtDesc.ColumnStats> columnStatsList1 = tableExtDesc1.getColumnStats();
- Assert.assertEquals(1, columnStatsList1.size());
-
- getInstance(getTestConfig()).removeTableExt("DEFAULT.WIDE_TABLE", "default");
- }
-
- @Test
- public void testTableExtCompatibility() throws IOException {
- String tableName = "DEFAULT.WIDE_TABLE";
- Map<String, String> oldTableExt = new HashMap<>();
- oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4");
- mockUpOldTableExtJson(tableName, oldTableExt);
- TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default");
- Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality());
- getInstance(getTestConfig()).removeTableExt(tableName, "default");
- }
-
- private void mockUpOldTableExtJson(String tableId, Map<String, String> tableExdProperties) throws IOException {
- String path = TableExtDesc.concatResourcePath(tableId, null);
-
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- JsonUtil.writeValueIndent(os, tableExdProperties);
- os.flush();
- InputStream is = new ByteArrayInputStream(os.toByteArray());
- getStore().putResource(path, is, System.currentTimeMillis());
- os.close();
- is.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/TableMetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/TableMetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/TableMetadataManagerTest.java
new file mode 100644
index 0000000..ff08ca1
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/TableMetadataManagerTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata;
+
+import static org.apache.kylin.metadata.TableMetadataManager.getInstance;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableExtDesc;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class TableMetadataManagerTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testListAllTables() throws Exception {
+ List<TableDesc> tables = getInstance(getTestConfig()).listAllTables(null);
+ Assert.assertNotNull(tables);
+ Assert.assertTrue(tables.size() > 0);
+ }
+
+ @Test
+ public void testFindTableByName() throws Exception {
+ TableDesc table = getInstance(getTestConfig()).getTableDesc("EDW.TEST_CAL_DT", "default");
+ Assert.assertNotNull(table);
+ Assert.assertEquals("EDW.TEST_CAL_DT", table.getIdentity());
+ }
+
+ @Test
+ public void testGetInstance() throws Exception {
+ Assert.assertNotNull(getInstance(getTestConfig()));
+ Assert.assertNotNull(getInstance(getTestConfig()).listAllTables(null));
+ Assert.assertTrue(getInstance(getTestConfig()).listAllTables(null).size() > 0);
+ }
+
+ @Test
+ public void testTableSample() throws IOException {
+ TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default");
+ Assert.assertNotNull(tableExtDesc);
+
+ List<TableExtDesc.ColumnStats> columnStatsList = new ArrayList<>();
+ TableExtDesc.ColumnStats columnStats = new TableExtDesc.ColumnStats();
+ columnStats.setColumnSamples("Max", "Min", "dfadsfdsfdsafds", "d");
+ columnStatsList.add(columnStats);
+ tableExtDesc.setColumnStats(columnStatsList);
+ getInstance(getTestConfig()).saveTableExt(tableExtDesc, "default");
+
+ TableExtDesc tableExtDesc1 = getInstance(getTestConfig()).getTableExt("DEFAULT.WIDE_TABLE", "default");
+ Assert.assertNotNull(tableExtDesc1);
+
+ List<TableExtDesc.ColumnStats> columnStatsList1 = tableExtDesc1.getColumnStats();
+ Assert.assertEquals(1, columnStatsList1.size());
+
+ getInstance(getTestConfig()).removeTableExt("DEFAULT.WIDE_TABLE", "default");
+ }
+
+ @Test
+ public void testTableExtCompatibility() throws IOException {
+ String tableName = "DEFAULT.WIDE_TABLE";
+ Map<String, String> oldTableExt = new HashMap<>();
+ oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4");
+ mockUpOldTableExtJson(tableName, oldTableExt);
+ TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName, "default");
+ Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality());
+ getInstance(getTestConfig()).removeTableExt(tableName, "default");
+ }
+
+ private void mockUpOldTableExtJson(String tableId, Map<String, String> tableExdProperties) throws IOException {
+ String path = TableExtDesc.concatResourcePath(tableId, null);
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, tableExdProperties);
+ os.flush();
+ InputStream is = new ByteArrayInputStream(os.toByteArray());
+ getStore().putResource(path, is, System.currentTimeMillis());
+ os.close();
+ is.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/draft/DraftManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/draft/DraftManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/draft/DraftManagerTest.java
index 153e82a..1b0fb40 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/draft/DraftManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/draft/DraftManagerTest.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -71,7 +71,7 @@ public class DraftManagerTest extends LocalFileMetadataTestCase {
}
private RootPersistentEntity getSampleModel() {
- MetadataManager metaMgr = MetadataManager.getInstance(getTestConfig());
+ DataModelManager metaMgr = DataModelManager.getInstance(getTestConfig());
DataModelDesc model = metaMgr.getDataModelDesc("ci_left_join_model");
return model;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
index f6d6fc3..7feefef 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelDescTest.java
@@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue;
import org.apache.commons.lang.ArrayUtils;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -45,19 +44,19 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
@Test
public void loadInnerModel() {
- DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_inner_join_model");
+ DataModelDesc model = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("ci_inner_join_model");
assertNotNull(model);
}
@Test
public void loadLeftModel() {
- DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("ci_left_join_model");
+ DataModelDesc model = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("ci_left_join_model");
assertNotNull(model);
}
@Test
public void testNoDupColInDimAndMeasure() {
- DataModelDesc model = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
+ DataModelDesc model = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
String[] metrics = model.getMetrics();
TblColRef col = model.findColumn("edw.test_cal_dt.cal_dt");
assertTrue(metrics.length == 2);
@@ -66,7 +65,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
@Test
public void testGetCopyOf() throws JsonProcessingException {
- DataModelDesc desc = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
+ DataModelDesc desc = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc");
DataModelDesc copyDesc = DataModelDesc.getCopyOf(desc);
// uuid is different, set to equals for json comparison
@@ -81,7 +80,7 @@ public class DataModelDescTest extends LocalFileMetadataTestCase {
@Test
public void testPartitionDescCopyOf() throws JsonProcessingException {
- PartitionDesc desc = MetadataManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc").partitionDesc;
+ PartitionDesc desc = DataModelManager.getInstance(getTestConfig()).getDataModelDesc("test_kylin_inner_join_model_desc").partitionDesc;
PartitionDesc copyDesc = PartitionDesc.getCopyOf(desc);
String descStr = JsonUtil.writeValueAsIndentString(desc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
new file mode 100644
index 0000000..ee75302
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DataModelManagerTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.metadata.model;
+
+import static org.apache.kylin.metadata.model.DataModelManager.getInstance;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class DataModelManagerTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCiModel() {
+ DataModelManager mgr = getInstance(getTestConfig());
+ DataModelDesc lm = mgr.getDataModelDesc("ci_left_join_model");
+ DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
+ assertSnowflakeQuality(lm);
+ assertSnowflakeQuality(im);
+
+ // check inner/left models are identical apart from the left/inner difference
+ assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
+ for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
+ JoinTableDesc lt = lm.getJoinTables()[i];
+ JoinTableDesc it = im.getJoinTables()[i];
+ assertEquals(lt.getAlias(), it.getAlias());
+ assertEquals(lt.getKind(), it.getKind());
+ assertEquals(lt.getTable(), it.getTable());
+ assertArrayEquals(lt.getJoin().getForeignKey(), it.getJoin().getForeignKey());
+ assertArrayEquals(lt.getJoin().getPrimaryKey(), it.getJoin().getPrimaryKey());
+ assertTrue(lt.getJoin().isLeftJoin());
+ assertTrue(it.getJoin().isInnerJoin());
+ }
+
+ assertEquals(lm.getDimensions().size(), im.getDimensions().size());
+ for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
+ ModelDimensionDesc ld = lm.getDimensions().get(i);
+ ModelDimensionDesc id = im.getDimensions().get(i);
+ assertEquals(ld.getTable(), id.getTable());
+ assertArrayEquals(ld.getColumns(), id.getColumns());
+ }
+
+ assertArrayEquals(lm.getMetrics(), im.getMetrics());
+ }
+
+ private void assertSnowflakeQuality(DataModelDesc model) {
+ Assert.assertNotNull(model);
+ try {
+ model.findTable("TEST_COUNTRY");
+ Assert.fail();
+ } catch (IllegalArgumentException ex) {
+ // excepted
+ }
+
+ Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+ Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+ Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+ Assert.assertNotNull(model.findColumn("BUYER_ID"));
+
+ }
+
+ @Test
+ public void testDataModel() throws Exception {
+ DataModelDesc modelDesc = getInstance(getTestConfig()).getDataModelDesc("test_kylin_left_join_model_desc");
+ Assert.assertTrue(modelDesc.getDimensions().size() > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
index d5e9de7..85ddb13 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -25,7 +25,6 @@ import java.util.Map.Entry;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.JoinsTree.Chain;
import org.junit.After;
import org.junit.Before;
@@ -47,7 +46,7 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() {
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc model = mgr.getDataModelDesc("ci_left_join_model");
JoinsTree joinsTree = model.getJoinsTree();
@@ -62,7 +61,7 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase {
@Test
public void testMatch() {
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc model = mgr.getDataModelDesc("ci_inner_join_model");
JoinsTree joinsTree = model.getJoinsTree();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
index 016c591..8f1a044 100644
--- a/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/source/datagen/DataGenTest.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -59,7 +59,7 @@ public class DataGenTest extends LocalFileMetadataTestCase {
}
private DataModelDesc getModel(String name) {
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc model = mgr.getDataModelDesc(name);
return model;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
index ea7422b..2f5c29d 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/CubeTupleConverter.java
@@ -35,7 +35,7 @@ import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureType.IAdvMeasureFiller;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -266,7 +266,7 @@ public class CubeTupleConverter implements ITupleConverter {
public LookupStringTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) {
long ts = System.currentTimeMillis();
- MetadataManager metaMgr = MetadataManager.getInstance(cubeSeg.getCubeInstance().getConfig());
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(cubeSeg.getCubeInstance().getConfig());
SnapshotManager snapshotMgr = SnapshotManager.getInstance(cubeSeg.getCubeInstance().getConfig());
String tableName = join.getPKSide().getTableIdentity();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index 0084670..b2a2ea3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -30,7 +30,7 @@ import org.apache.kylin.engine.mr.IMROutput.IMRBatchCubingOutputSide;
import org.apache.kylin.engine.mr.IMROutput.IMRBatchMergeOutputSide;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2;
import org.apache.kylin.engine.mr.IMROutput2.IMRBatchMergeOutputSide2;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.SourceFactory;
@@ -53,7 +53,7 @@ public class MRUtil {
}
private static TableDesc getTableDesc(String tableName, String prj) {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(tableName, prj);
+ return TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc(tableName, prj);
}
public static IMRBatchCubingOutputSide getBatchCubingOutputSide(CubeSegment seg) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
index a1815e2..4d92f8e 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapperTest.java
@@ -39,9 +39,9 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.source.IReadableTable.TableSignature;
@@ -90,7 +90,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
logger.info("The metadataUrl is : " + getTestConfig());
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
CubeManager.clearCache();
ProjectManager.clearCache();
DictionaryManager.clearCache();
[10/11] kylin git commit: update KYLIN version to 2.3.0
Posted by li...@apache.org.
update KYLIN version to 2.3.0
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/dbc513ee
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/dbc513ee
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/dbc513ee
Branch: refs/heads/master
Commit: dbc513eef80d6e7c43f6f14fb69586c6b8897f3a
Parents: 9295947
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 2 17:14:11 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:31:56 2017 +0800
----------------------------------------------------------------------
assembly/pom.xml | 2 +-
atopcalcite/pom.xml | 2 +-
core-common/pom.xml | 2 +-
.../src/main/java/org/apache/kylin/common/KylinVersion.java | 2 +-
core-cube/pom.xml | 2 +-
core-dictionary/pom.xml | 2 +-
core-job/pom.xml | 2 +-
core-metadata/pom.xml | 2 +-
core-storage/pom.xml | 2 +-
engine-mr/pom.xml | 2 +-
engine-spark/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
kylin-it/pom.xml | 2 +-
pom.xml | 5 ++---
query/pom.xml | 2 +-
server-base/pom.xml | 2 +-
server/pom.xml | 2 +-
source-hive/pom.xml | 2 +-
source-kafka/pom.xml | 2 +-
storage-hbase/pom.xml | 2 +-
tomcat-ext/pom.xml | 2 +-
tool-assembly/pom.xml | 2 +-
tool/pom.xml | 2 +-
23 files changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 3435ed4..a0e7bd8 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -27,7 +27,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 6542836..80b3a9a 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 6de4897..c6fc893 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index e52feab..749cb86 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -95,7 +95,7 @@ public class KylinVersion implements Comparable {
/**
* Require MANUAL updating kylin version per ANY upgrading.
*/
- private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.2.0.20500");
+ private static final KylinVersion CURRENT_KYLIN_VERSION = new KylinVersion("2.3.0.20500");
private static final KylinVersion VERSION_200 = new KylinVersion("2.0.0");
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 7b79608..3712f88 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-dictionary/pom.xml
----------------------------------------------------------------------
diff --git a/core-dictionary/pom.xml b/core-dictionary/pom.xml
index 40959f8..3a53149 100644
--- a/core-dictionary/pom.xml
+++ b/core-dictionary/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index a7d31b0..1f433bf 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 469f880..19ef387 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -30,7 +30,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 1808cd2..ce913b5 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/engine-mr/pom.xml
----------------------------------------------------------------------
diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml
index 3bfa188..1ddc2bd 100644
--- a/engine-mr/pom.xml
+++ b/engine-mr/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 56cd9ce..e2df5a6 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index ad85412..5374226 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/kylin-it/pom.xml
----------------------------------------------------------------------
diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml
index fa296c7..0d97100 100644
--- a/kylin-it/pom.xml
+++ b/kylin-it/pom.xml
@@ -31,7 +31,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1f05806..54cdf08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
<packaging>pom</packaging>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>Apache Kylin</name>
<url>http://kylin.apache.org</url>
@@ -1002,8 +1002,7 @@
<phase>validate</phase>
<configuration>
<configLocation>dev-support/checkstyle.xml</configLocation>
- <suppressionsLocation>dev-support/checkstyle-suppressions.xml
- </suppressionsLocation>
+ <suppressionsLocation>dev-support/checkstyle-suppressions.xml</suppressionsLocation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
<consoleOutput>true</consoleOutput>
<failsOnError>true</failsOnError>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 4ece9ef..054927c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index 24a0da8..07934ed 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -30,7 +30,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index d45150d..e1a44ce 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/source-hive/pom.xml
----------------------------------------------------------------------
diff --git a/source-hive/pom.xml b/source-hive/pom.xml
index 877665b..9a4d537 100644
--- a/source-hive/pom.xml
+++ b/source-hive/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index 97c2312..f9197d4 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -29,7 +29,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index c9072c3..15791d5 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -30,7 +30,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/tomcat-ext/pom.xml
----------------------------------------------------------------------
diff --git a/tomcat-ext/pom.xml b/tomcat-ext/pom.xml
index 6540d00..3805e92 100644
--- a/tomcat-ext/pom.xml
+++ b/tomcat-ext/pom.xml
@@ -30,7 +30,7 @@
<parent>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/tool-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/tool-assembly/pom.xml b/tool-assembly/pom.xml
index 9bef1f8..d2a2df3 100644
--- a/tool-assembly/pom.xml
+++ b/tool-assembly/pom.xml
@@ -29,7 +29,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/kylin/blob/dbc513ee/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 62eaf16..124f25e 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -29,7 +29,7 @@
<parent>
<artifactId>kylin</artifactId>
<groupId>org.apache.kylin</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<properties>
[09/11] kylin git commit: KYLIN-2921 Refactor DataModelDesc,
split DataModelManager
Posted by li...@apache.org.
KYLIN-2921 Refactor DataModelDesc, split DataModelManager
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f45d8133
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f45d8133
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f45d8133
Branch: refs/heads/master
Commit: f45d8133b28f0063f1d73087f6dd65b34cf372cf
Parents: 1f6e04c
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 2 17:45:38 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:31:56 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 10 +-
.../streaming/StreamingTableDataGenerator.java | 4 +-
.../apache/kylin/common/KylinConfigBase.java | 8 +
.../java/org/apache/kylin/cube/CubeManager.java | 14 +-
.../kylin/cube/cli/CubeSignatureRefresher.java | 12 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 4 +-
.../kylin/cube/model/v1_4_0/CubeDesc.java | 4 +-
.../upgrade/common/CubeMetadataUpgrade.java | 8 +-
.../v1_4_0/CubeMetadataUpgrade_v_1_4_0.java | 11 +-
.../kylin/cube/AggregationGroupRuleTest.java | 4 +-
.../apache/kylin/cube/CubeManagerCacheTest.java | 4 +-
.../kylin/cube/common/RowKeySplitterTest.java | 4 +-
.../kylin/cube/cuboid/CuboidSchedulerTest.java | 4 +-
.../apache/kylin/cube/cuboid/CuboidTest.java | 4 +-
.../apache/kylin/cube/kv/RowKeyDecoderTest.java | 4 +-
.../apache/kylin/cube/kv/RowKeyEncoderTest.java | 4 +-
.../apache/kylin/dict/DictionaryManager.java | 16 +-
.../apache/kylin/dict/lookup/SnapshotCLI.java | 4 +-
.../kylin/dict/lookup/SnapshotManager.java | 12 +-
.../kylin/dict/DictionaryManagerTest.java | 4 +-
.../kylin/dict/lookup/LookupTableTest.java | 4 +-
.../org/apache/kylin/job/dao/ExecutableDao.java | 4 +-
.../apache/kylin/metadata/MetadataManager.java | 706 -------------------
.../kylin/metadata/TableMetadataManager.java | 494 +++++++++++++
.../metadata/filter/ColumnTupleFilter.java | 4 +-
.../metadata/filter/UDF/MassInTupleFilter.java | 4 +-
.../kylin/metadata/model/DataModelDesc.java | 125 +---
.../kylin/metadata/model/DataModelManager.java | 331 +++++++++
.../kylin/metadata/project/ProjectInstance.java | 7 -
.../kylin/metadata/project/ProjectL2Cache.java | 10 +-
.../kylin/metadata/project/ProjectManager.java | 29 +-
.../kylin/source/datagen/ColumnGenConfig.java | 4 +-
.../source/datagen/ModelDataGenerator.java | 4 +-
.../kylin/metadata/MetadataManagerTest.java | 180 -----
.../metadata/TableMetadataManagerTest.java | 119 ++++
.../kylin/metadata/draft/DraftManagerTest.java | 4 +-
.../kylin/metadata/model/DataModelDescTest.java | 11 +-
.../metadata/model/DataModelManagerTest.java | 100 +++
.../kylin/metadata/model/JoinsTreeTest.java | 5 +-
.../kylin/source/datagen/DataGenTest.java | 4 +-
.../storage/gtrecord/CubeTupleConverter.java | 4 +-
.../java/org/apache/kylin/engine/mr/MRUtil.java | 4 +-
.../engine/mr/steps/MergeCuboidMapperTest.java | 4 +-
.../localmeta/cube_desc/ci_inner_join_cube.json | 77 +-
.../localmeta/cube_desc/ci_left_join_cube.json | 77 +-
.../cube_desc/ut_inner_join_cube_partial.json | 41 +-
.../model_desc/ci_inner_join_model.json | 59 +-
.../model_desc/ci_left_join_model.json | 59 +-
.../java/org/apache/kylin/query/H2Database.java | 4 +-
.../apache/kylin/query/ITKylinQueryTest.java | 6 -
.../source/hive/ITSnapshotManagerTest.java | 4 +-
.../jdbc/ITJdbcSourceTableLoaderTest.java | 4 +-
.../source/jdbc/ITJdbcTableReaderTest.java | 4 +-
.../query/sql_computedcolumn/query01.sql | 14 -
.../sql_computedcolumn/query01.sql.compare | 14 -
.../query/sql_computedcolumn/query02.sql | 21 -
.../sql_computedcolumn/query02.sql.compare | 21 -
.../query/sql_computedcolumn/query03.sql | 21 -
.../sql_computedcolumn/query03.sql.compare | 21 -
.../query/sql_computedcolumn/query04.sql | 27 -
.../sql_computedcolumn/query04.sql.compare | 26 -
.../query/sql_verifyCount/query01.sql.expected | 2 +-
.../query/sql_verifyCount/query02.sql.expected | 2 +-
.../query/sql_verifyCount/query03.sql.expected | 2 +-
.../query/sql_verifyCount/query04.sql.expected | 2 +-
.../query/sql_verifyCount/query10.sql.expected | 2 +-
.../query/sql_verifyCount/query11.sql.expected | 2 +-
.../.settings/org.eclipse.core.resources.prefs | 1 +
.../org/apache/kylin/query/QueryConnection.java | 7 +-
.../apache/kylin/query/schema/OLAPSchema.java | 9 +-
.../kylin/query/schema/OLAPSchemaFactory.java | 2 -
.../query/security/QueryInterceptorUtil.java | 120 ----
.../kylin/rest/controller/ModelController.java | 6 +-
.../rest/controller/ModelDescController.java | 4 +-
.../apache/kylin/rest/job/HybridCubeCLI.java | 6 +-
.../rest/response/DataModelDescResponse.java | 1 -
.../kylin/rest/security/TableInterceptor.java | 17 +-
.../apache/kylin/rest/service/BasicService.java | 11 +-
.../apache/kylin/rest/service/CacheService.java | 5 -
.../apache/kylin/rest/service/CubeService.java | 2 +-
.../kylin/rest/service/ExtFilterService.java | 12 +-
.../kylin/rest/service/HybridService.java | 2 +-
.../apache/kylin/rest/service/ModelService.java | 116 +--
.../rest/service/TableSchemaUpdateChecker.java | 6 +-
.../apache/kylin/rest/service/TableService.java | 24 +-
.../kylin/rest/service/CacheServiceTest.java | 37 +-
.../kylin/rest/service/CubeServiceTest.java | 2 +-
.../kylin/rest/service/JobServiceTest.java | 2 +-
.../kylin/rest/service/ModelServiceTest.java | 77 +-
.../kylin/rest/service/QueryServiceTest.java | 2 +-
.../apache/kylin/source/hive/HiveMRInput.java | 4 +-
.../kylin/source/hive/HiveMetadataExplorer.java | 4 +-
.../cardinality/ColumnCardinalityMapper.java | 4 +-
.../cardinality/HiveColumnCardinalityJob.java | 4 +-
.../HiveColumnCardinalityUpdateJob.java | 4 +-
.../kylin/storage/hbase/HBaseStorage.java | 4 +-
.../hbase/util/ExtendCubeToHybridCLI.java | 6 +-
.../cube/MeasureTypeOnlyAggrInBaseTest.java | 4 +-
.../hbase/steps/RowValueDecoderTest.java | 4 +-
.../apache/kylin/tool/CubeMetaExtractor.java | 6 +-
.../org/apache/kylin/tool/CubeMetaIngester.java | 38 +-
.../kylin/tool/ExtendCubeToHybridCLI.java | 6 +-
.../apache/kylin/tool/KylinLogExtractor.java | 6 +-
.../apache/kylin/tool/CubeMetaIngesterTest.java | 6 +-
104 files changed, 1362 insertions(+), 2021 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index bdf7c47..5866c09 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -41,8 +41,9 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.job.streaming.StreamDataLoader;
import org.apache.kylin.job.streaming.StreamingTableDataGenerator;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.ISampleDataDeployer;
@@ -127,7 +128,7 @@ public class DeployUtil {
System.out.println("build cube with random dataset");
// data is generated according to cube descriptor and saved in resource store
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc(modelName), 10000);
gen.generate();
} else {
@@ -190,8 +191,9 @@ public class DeployUtil {
private static void deployTables(String modelName) throws Exception {
- MetadataManager metaMgr = MetadataManager.getInstance(config());
- DataModelDesc model = metaMgr.getDataModelDesc(modelName);
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(config());
+ DataModelManager modelMgr = DataModelManager.getInstance(config());
+ DataModelDesc model = modelMgr.getDataModelDesc(modelName);
Set<TableRef> tables = model.getAllTables();
Set<String> TABLE_NAMES = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
index 2144c3a..1fef07e 100644
--- a/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
+++ b/assembly/src/test/java/org/apache/kylin/job/streaming/StreamingTableDataGenerator.java
@@ -23,7 +23,7 @@ import java.util.List;
import java.util.Random;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -51,7 +51,7 @@ public class StreamingTableDataGenerator {
Preconditions.checkArgument(recordCount > 0);
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName, prj);
+ TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, prj);
SortedMultiset<Long> times = TreeMultiset.create();
Random r = new Random();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ee05d69..f362884 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -275,7 +275,15 @@ abstract public class KylinConfigBase implements Serializable {
r.putAll(getPropertiesByPrefix("kylin.metadata.resource-store-provider.")); // note the naming convention -- http://kylin.apache.org/development/coding_naming_convention.html
return r;
}
+
+ public String getDataModelImpl() {
+ return getOptional("kylin.metadata.data-model-impl", null);
+ }
+ public String getDataModelManagerImpl() {
+ return getOptional("kylin.metadata.data-model-manager-impl", null);
+ }
+
public String[] getRealizationProviders() {
return getOptionalStringArray("kylin.metadata.realization-providers", //
new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 33dc312..3bb9f21 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -51,7 +51,7 @@ import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.dict.lookup.LookupStringTable;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
@@ -183,7 +183,6 @@ public class CubeManager implements IRealizationProvider {
}
public CubeInstance getCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
return cubeMap.get(cubeName);
}
@@ -205,7 +204,6 @@ public class CubeManager implements IRealizationProvider {
*/
public List<CubeInstance> getCubesByDesc(String descName) {
- descName = descName.toUpperCase();
List<CubeInstance> list = listAllCubes();
List<CubeInstance> result = new ArrayList<CubeInstance>();
Iterator<CubeInstance> it = list.iterator();
@@ -279,7 +277,7 @@ public class CubeManager implements IRealizationProvider {
}
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
SnapshotManager snapshotMgr = getSnapshotManager();
TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, cubeSeg.getProject()));
@@ -644,7 +642,7 @@ public class CubeManager implements IRealizationProvider {
try {
SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject());
+ TableDesc tableDesc = getTableManager().getTableDesc(tableName, cubeSegment.getProject());
return new LookupStringTable(tableDesc, pkCols, snapshot);
} catch (IOException e) {
throw new IllegalStateException(
@@ -808,10 +806,10 @@ public class CubeManager implements IRealizationProvider {
&& (cubeName.startsWith("test_kylin_cube") || cubeName.startsWith("test_streaming"));
}
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
+ private TableMetadataManager getTableManager() {
+ return TableMetadataManager.getInstance(config);
}
-
+
private DictionaryManager getDictionaryManager() {
return DictionaryManager.getInstance(config);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java
index 1db804f..d07c93b 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/CubeSignatureRefresher.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.cli;
+import java.io.IOException;
import java.util.List;
import org.apache.commons.lang.ArrayUtils;
@@ -26,7 +27,8 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,8 +77,12 @@ public class CubeSignatureRefresher {
}
private void verify() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
+ try {
+ Broadcaster.getInstance(config).notifyClearAll();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ DataModelManager.getInstance(config);
CubeDescManager.getInstance(config);
CubeManager.getInstance(config);
ProjectManager.getInstance(config);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 827ac81..6a75083 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -56,7 +56,6 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
@@ -65,6 +64,7 @@ import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -583,7 +583,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
"Too many rowkeys (%s) in CubeDesc, please try to reduce dimension number or adopt derived dimensions",
this.rowkey.getRowKeyColumns().length);
- this.model = MetadataManager.getInstance(config).getDataModelDesc(modelName);
+ this.model = DataModelManager.getInstance(config).getDataModelDesc(modelName);
checkNotNull(this.model, "DateModelDesc(%s) not found", modelName);
for (DimensionDesc dim : dimensions) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
index f1a57e0..f5c4a0d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/v1_4_0/CubeDesc.java
@@ -43,13 +43,13 @@ import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -454,7 +454,7 @@ public class CubeDesc extends RootPersistentEntity {
this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
}
- this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
+ this.model = DataModelManager.getInstance(config).getDataModelDesc(this.modelName);
if (this.model == null) {
this.addError("No data model found with name '" + modelName + "'.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java
index f70286d..f2f6b57 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/common/CubeMetadataUpgrade.java
@@ -29,7 +29,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,7 +63,7 @@ public abstract class CubeMetadataUpgrade {
}
public void clear() {
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
ProjectManager.clearCache();
@@ -73,8 +73,8 @@ public abstract class CubeMetadataUpgrade {
logger.info("=================================================================");
logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:");
logger.info("=================================================================");
- MetadataManager.clearCache();
- MetadataManager.getInstance(config);
+ DataModelManager.clearCache();
+ DataModelManager.getInstance(config);
CubeDescManager.clearCache();
CubeDescManager.getInstance(config);
CubeManager.clearCache();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
index 29465f7..1e90910 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v1_4_0/CubeMetadataUpgrade_v_1_4_0.java
@@ -32,8 +32,9 @@ import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.cube.model.v1_4_0.CubeDesc;
import org.apache.kylin.cube.model.v1_4_0.DimensionDesc;
import org.apache.kylin.cube.upgrade.common.CubeMetadataUpgrade;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.apache.kylin.metadata.model.PartitionDesc;
@@ -82,8 +83,8 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade {
}
private DataModelDesc getDataModelDesc(String modelName) {
- MetadataManager.clearCache();
- return MetadataManager.getInstance(config).getDataModelDesc(modelName);
+ DataModelManager.clearCache();
+ return DataModelManager.getInstance(config).getDataModelDesc(modelName);
}
public void dowork() {
@@ -91,7 +92,7 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade {
for (String path : paths) {
logger.info("CubeMetadataUpgrade_v_1_4_0 handling in dowork {}", path);
CubeDesc cubeDesc = loadOldCubeDesc(path);
- cubeDesc.init(config, MetadataManager.getInstance(config).getAllTablesMap(cubeDesc.getProject()));
+ cubeDesc.init(config, TableMetadataManager.getInstance(config).getAllTablesMap(cubeDesc.getProject()));
upgradeDataModelDesc(cubeDesc);
upgradeCubeDesc(cubeDesc);
@@ -144,7 +145,7 @@ public class CubeMetadataUpgrade_v_1_4_0 extends CubeMetadataUpgrade {
}
if (upgrade) {
- store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER);
+ store.putResource(modelDesc.getResourcePath(), modelDesc, DataModelManager.getInstance(config).getDataModelSerializer());
updatedResources.add(modelDesc.getResourcePath());
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
index 62b9c98..1444ee1 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/AggregationGroupRuleTest.java
@@ -32,7 +32,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.IValidatorRule;
import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.cube.model.validation.rule.AggregationGroupRule;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -41,7 +41,7 @@ public class AggregationGroupRuleTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index de3048f..52b9042 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.junit.After;
@@ -41,7 +41,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
CubeManager.clearCache();
ProjectManager.clearCache();
cubeManager = CubeManager.getInstance(getTestConfig());
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
index b41f23b..6d32586 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/common/RowKeySplitterTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -33,7 +33,7 @@ public class RowKeySplitterTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
index 3f3a7c4..09200b8 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidSchedulerTest.java
@@ -31,7 +31,7 @@ import java.util.Set;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -47,7 +47,7 @@ public class CuboidSchedulerTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
index 0fa849e..0a77bdc 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/CuboidTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -61,7 +61,7 @@ public class CuboidTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
index ec1f221..459e734 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyDecoderTest.java
@@ -29,7 +29,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -40,7 +40,7 @@ public class RowKeyDecoderTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
index 5af8d8a..dcd883e 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/kv/RowKeyEncoderTest.java
@@ -29,7 +29,7 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.cuboid.Cuboid;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -40,7 +40,7 @@ public class RowKeyEncoderTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 0eab8e4..e97899c 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -34,8 +34,8 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.source.IReadableTable;
import org.apache.kylin.source.IReadableTable.TableSignature;
@@ -176,7 +176,7 @@ public class DictionaryManager {
}
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ResourceStore store = DataModelManager.getInstance(config).getStore();
NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir());
if (existings == null)
return null;
@@ -343,7 +343,7 @@ public class DictionaryManager {
}
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
- final ResourceStore store = MetadataManager.getInstance(config).getStore();
+ final ResourceStore store = DataModelManager.getInstance(config).getStore();
final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
TableSignature input = dictInfo.getInput();
@@ -357,7 +357,7 @@ public class DictionaryManager {
}
private DictionaryInfo findLargestDictInfo(DictionaryInfo dictInfo) throws IOException {
- final ResourceStore store = MetadataManager.getInstance(config).getStore();
+ final ResourceStore store = DataModelManager.getInstance(config).getStore();
final List<DictionaryInfo> allResources = store.getAllResources(dictInfo.getResourceDir(), DictionaryInfo.class, DictionaryInfoSerializer.INFO_SERIALIZER);
DictionaryInfo largestDict = null;
@@ -376,7 +376,7 @@ public class DictionaryManager {
public void removeDictionary(String resourcePath) throws IOException {
logger.info("Remvoing dict: " + resourcePath);
- ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ResourceStore store = DataModelManager.getInstance(config).getStore();
store.deleteResource(resourcePath);
dictCache.invalidate(resourcePath);
}
@@ -386,7 +386,7 @@ public class DictionaryManager {
info.setSourceTable(srcTable);
info.setSourceColumn(srcCol);
- ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ResourceStore store = DataModelManager.getInstance(config).getStore();
NavigableSet<String> existings = store.listResources(info.getResourceDir());
if (existings == null)
return;
@@ -396,7 +396,7 @@ public class DictionaryManager {
}
void save(DictionaryInfo dict) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ResourceStore store = DataModelManager.getInstance(config).getStore();
String path = dict.getResourcePath();
logger.info("Saving dictionary at " + path);
@@ -412,7 +412,7 @@ public class DictionaryManager {
}
DictionaryInfo load(String resourcePath, boolean loadDictObj) throws IOException {
- ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ResourceStore store = DataModelManager.getInstance(config).getStore();
logger.info("DictionaryManager(" + System.identityHashCode(this) + ") loading DictionaryInfo(loadDictObj:" + loadDictObj + ") at " + resourcePath);
DictionaryInfo info = store.getResource(resourcePath, DictionaryInfo.class, loadDictObj ? DictionaryInfoSerializer.FULL_SERIALIZER : DictionaryInfoSerializer.INFO_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
index 0fe102b..2093d23 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotCLI.java
@@ -21,7 +21,7 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.SourceFactory;
@@ -35,7 +35,7 @@ public class SnapshotCLI {
private static void rebuild(String table, String overwriteUUID, String project) throws IOException {
KylinConfig conf = KylinConfig.getInstanceFromEnv();
- MetadataManager metaMgr = MetadataManager.getInstance(conf);
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(conf);
SnapshotManager snapshotMgr = SnapshotManager.getInstance(conf);
TableDesc tableDesc = metaMgr.getTableDesc(table, project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index b997a88..c10deb4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.IReadableTable;
import org.apache.kylin.source.IReadableTable.TableSignature;
@@ -116,7 +116,7 @@ public class SnapshotManager {
}
public void removeSnapshot(String resourcePath) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
+ ResourceStore store = DataModelManager.getInstance(this.config).getStore();
store.deleteResource(resourcePath);
snapshotCache.invalidate(resourcePath);
}
@@ -171,7 +171,7 @@ public class SnapshotManager {
}
private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
+ ResourceStore store = DataModelManager.getInstance(this.config).getStore();
String resourceDir = snapshot.getResourceDir();
NavigableSet<String> existings = store.listResources(resourceDir);
if (existings == null)
@@ -189,7 +189,7 @@ public class SnapshotManager {
}
private String checkDupByContent(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
+ ResourceStore store = DataModelManager.getInstance(this.config).getStore();
String resourceDir = snapshot.getResourceDir();
NavigableSet<String> existings = store.listResources(resourceDir);
if (existings == null)
@@ -205,14 +205,14 @@ public class SnapshotManager {
}
private void save(SnapshotTable snapshot) throws IOException {
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
+ ResourceStore store = DataModelManager.getInstance(this.config).getStore();
String path = snapshot.getResourcePath();
store.putResource(path, snapshot, SnapshotTableSerializer.FULL_SERIALIZER);
}
private SnapshotTable load(String resourcePath, boolean loadData) throws IOException {
logger.info("Loading snapshotTable from " + resourcePath + ", with loadData: " + loadData);
- ResourceStore store = MetadataManager.getInstance(this.config).getStore();
+ ResourceStore store = DataModelManager.getInstance(this.config).getStore();
SnapshotTable table = store.getResource(resourcePath, SnapshotTable.class, loadData ? SnapshotTableSerializer.FULL_SERIALIZER : SnapshotTableSerializer.INFO_SERIALIZER);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
index 9c126b4..6a86e33 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DictionaryManagerTest.java
@@ -26,8 +26,8 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Before;
@@ -50,7 +50,7 @@ public class DictionaryManagerTest extends LocalFileMetadataTestCase {
public void testBuildSaveDictionary() throws IOException {
KylinConfig config = KylinConfig.getInstanceFromEnv();
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- MetadataManager metaMgr = MetadataManager.getInstance(config);
+ DataModelManager metaMgr = DataModelManager.getInstance(config);
DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc");
TblColRef col = model.findColumn("lstg_format_name");
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index 57d70cf..f53e8fe 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -28,7 +28,7 @@ import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.dict.TrieDictionaryForest;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.junit.After;
import org.junit.Assert;
@@ -121,7 +121,7 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
public LookupTable<String> initLookupTable() throws Exception {
- MetadataManager metaMgr = MetadataManager.getInstance(config);
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
String tableName = "EDW.TEST_CAL_DT";
String[] pkCols = new String[] { "CAL_DT" };
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
index bd020f8..16875b1 100644
--- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
+++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java
@@ -31,7 +31,7 @@ import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.job.exception.PersistentException;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,7 +73,7 @@ public class ExecutableDao {
private ExecutableDao(KylinConfig config) {
logger.info("Using metadata url: " + config);
- this.store = MetadataManager.getInstance(config).getStore();
+ this.store = DataModelManager.getInstance(config).getStore();
}
private String pathOfJob(ExecutablePO job) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
deleted file mode 100644
index bfc7f08..0000000
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ /dev/null
@@ -1,706 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.metadata;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.RawResource;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
-import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.ExternalFilterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TableExtDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Serves (and caches) metadata for Kylin instance.
- * <p/>
- * Also provides a ResourceStore for general purpose data persistence.
- * Metadata is serialized as JSON and stored in ResourceStore.
- *
- * @author yangli9
- */
-public class MetadataManager {
-
- private static final Logger logger = LoggerFactory.getLogger(MetadataManager.class);
-
- public static final Serializer<TableDesc> TABLE_SERIALIZER = new JsonSerializer<TableDesc>(TableDesc.class);
- public static final Serializer<TableExtDesc> TABLE_EXT_SERIALIZER = new JsonSerializer<TableExtDesc>(
- TableExtDesc.class);
- public static final Serializer<DataModelDesc> MODELDESC_SERIALIZER = new JsonSerializer<DataModelDesc>(
- DataModelDesc.class);
- public static final Serializer<ExternalFilterDesc> EXTERNAL_FILTER_DESC_SERIALIZER = new JsonSerializer<ExternalFilterDesc>(
- ExternalFilterDesc.class);
-
- // static cached instances
- private static final ConcurrentMap<KylinConfig, MetadataManager> CACHE = new ConcurrentHashMap<KylinConfig, MetadataManager>();
-
- public static MetadataManager getInstance(KylinConfig config) {
- MetadataManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (MetadataManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new MetadataManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist, current keys: {}", StringUtils
- .join(Iterators.transform(CACHE.keySet().iterator(), new Function<KylinConfig, String>() {
- @Nullable
- @Override
- public String apply(@Nullable KylinConfig input) {
- return String.valueOf(System.identityHashCode(input));
- }
- }), ","));
- }
-
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init MetadataManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // table name ==> SourceTable
- private CaseInsensitiveStringCache<TableDesc> srcTableMap;
- // name => value
- private CaseInsensitiveStringCache<TableExtDesc> srcTableExtMap;
- // name => DataModelDesc
- private CaseInsensitiveStringCache<DataModelDesc> dataModelDescMap;
- // name => External Filter Desc
- private CaseInsensitiveStringCache<ExternalFilterDesc> extFilterMap;
-
- private MetadataManager(KylinConfig config) throws IOException {
- init(config);
- }
-
- /**
- * Tell MetadataManager that the instance has changed. The cube info will
- * be stored Reload the cube desc and source table A broadcast must be sent
- * out
- *
- * @return
- * @throws IOException
- */
- public void reload() {
- clearCache();
- getInstance(config);
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- public List<DataModelDesc> listDataModels() {
- return Lists.newArrayList(this.dataModelDescMap.values());
- }
-
- public List<TableDesc> listAllTables(String prj) {
- return Lists.newArrayList(getAllTablesMap(prj).values());
- }
-
- public List<ExternalFilterDesc> listAllExternalFilters() {
- return Lists.newArrayList(extFilterMap.values());
- }
-
- public Map<String, TableDesc> getAllTablesMap(String prj) {
- Map<String, TableDesc> globalTables = new LinkedHashMap<>();
- Map<String, TableDesc> projectTables = new LinkedHashMap<>();
-
- for (TableDesc t : srcTableMap.values()) {
- if (t.getProject() == null)
- globalTables.put(t.getIdentity(), t);
- else if (t.getProject().equals(prj))
- projectTables.put(t.getIdentity(), t);
- }
-
- Map<String, TableDesc> result = globalTables;
- result.putAll(projectTables);
- return result;
- }
-
- /**
- * Get TableDesc by name
- */
- public TableDesc getTableDesc(String tableName, String prj) {
- if (tableName.indexOf(".") < 0)
- tableName = "DEFAULT." + tableName;
-
- tableName.toUpperCase();
-
- TableDesc result = srcTableMap.get(mapKey(tableName, prj));
- if (result == null)
- result = srcTableMap.get(mapKey(tableName, null));
-
- return result;
- }
-
- public ExternalFilterDesc getExtFilterDesc(String filterTableName) {
- ExternalFilterDesc result = extFilterMap.get(filterTableName);
- return result;
- }
-
- /**
- * Get table extended info. Keys are defined in {@link MetadataConstants}
- *
- * @param tableName
- * @return
- */
- public TableExtDesc getTableExt(String tableName, String prj) {
- TableDesc t = getTableDesc(tableName, prj);
- if (t == null)
- return null;
-
- return getTableExt(t);
- }
-
- public TableExtDesc getTableExt(TableDesc t) {
- TableExtDesc result = srcTableExtMap.get(mapKey(t.getIdentity(), t.getProject()));
-
- // avoid returning null, since the TableDesc exists
- if (null == result) {
- result = new TableExtDesc();
- result.setIdentity(t.getIdentity());
- result.setUuid(UUID.randomUUID().toString());
- result.setLastModified(0);
- result.init(t.getProject());
- srcTableExtMap.put(mapKey(t.getIdentity(), t.getProject()), result);
- }
- return result;
- }
-
- public void saveTableExt(TableExtDesc tableExt, String prj) throws IOException {
- if (tableExt.getUuid() == null || tableExt.getIdentity() == null) {
- throw new IllegalArgumentException();
- }
-
- // updating a legacy global table
- if (tableExt.getProject() == null) {
- if (getTableExt(tableExt.getIdentity(), prj).getProject() != null)
- throw new IllegalStateException(
- "Updating a legacy global TableExtDesc while a project level version exists: "
- + tableExt.getIdentity() + ", " + prj);
- prj = tableExt.getProject();
- }
-
- tableExt.init(prj);
-
- String path = TableExtDesc.concatResourcePath(tableExt.getIdentity(), prj);
-
- ResourceStore store = getStore();
-
- TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
- if (t != null && t.getIdentity() == null)
- store.deleteResource(path);
-
- store.putResource(path, tableExt, TABLE_EXT_SERIALIZER);
- srcTableExtMap.put(mapKey(tableExt.getIdentity(), tableExt.getProject()), tableExt);
- }
-
- public void removeTableExt(String tableName, String prj) throws IOException {
- // note, here assume always delete TableExtDesc first, then TableDesc
- TableExtDesc t = getTableExt(tableName, prj);
- if (t == null)
- return;
-
- String path = TableExtDesc.concatResourcePath(t.getIdentity(), t.getProject());
- getStore().deleteResource(path);
- srcTableExtMap.remove(mapKey(t.getIdentity(), t.getProject()));
- }
-
- public void saveSourceTable(TableDesc srcTable, String prj) throws IOException {
- if (srcTable.getUuid() == null || srcTable.getIdentity() == null) {
- throw new IllegalArgumentException();
- }
-
- srcTable.init(prj);
-
- String path = TableDesc.concatResourcePath(srcTable.getIdentity(), prj);
- getStore().putResource(path, srcTable, TABLE_SERIALIZER);
-
- srcTableMap.put(mapKey(srcTable.getIdentity(), prj), srcTable);
- }
-
- public void removeSourceTable(String tableIdentity, String prj) throws IOException {
- TableDesc t = getTableDesc(tableIdentity, prj);
- if (t == null)
- return;
-
- String path = TableDesc.concatResourcePath(t.getIdentity(), t.getProject());
- getStore().deleteResource(path);
- srcTableMap.remove(mapKey(t.getIdentity(), t.getProject()));
- }
-
- public void saveExternalFilter(ExternalFilterDesc desc) throws IOException {
- if (desc.getUuid() == null) {
- throw new IllegalArgumentException("UUID not set.");
- }
- String path = desc.getResourcePath();
- getStore().putResource(path, desc, EXTERNAL_FILTER_DESC_SERIALIZER);
- desc = reloadExternalFilterAt(path);
- extFilterMap.put(desc.getName(), desc);
-
- }
-
- public void removeExternalFilter(String name) throws IOException {
- String path = ExternalFilterDesc.concatResourcePath(name);
- getStore().deleteResource(path);
- extFilterMap.remove(name);
-
- }
-
- private void init(KylinConfig config) throws IOException {
- this.config = config;
- this.srcTableMap = new CaseInsensitiveStringCache<>(config, "table");
- this.srcTableExtMap = new CaseInsensitiveStringCache<>(config, "table_ext");
- this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model");
- this.extFilterMap = new CaseInsensitiveStringCache<>(config, "external_filter");
-
- reloadAllSourceTable();
- reloadAllTableExt();
- reloadAllDataModel();
- reloadAllExternalFilter();
-
- // touch lower level metadata before registering my listener
- Broadcaster.getInstance(config).registerListener(new SrcTableSyncListener(), "table");
- Broadcaster.getInstance(config).registerListener(new SrcTableExtSyncListener(), "table_ext");
- Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model");
- Broadcaster.getInstance(config).registerListener(new ExtFilterSyncListener(), "external_filter");
- }
-
- private class SrcTableSyncListener extends Broadcaster.Listener {
- @Override
- public void onClearAll(Broadcaster broadcaster) throws IOException {
- clearCache();
- }
-
- @Override
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
- throws IOException {
- if (event == Event.DROP)
- srcTableMap.removeLocal(cacheKey);
- else
- reloadSourceTableAt(TableDesc.concatRawResourcePath(cacheKey));
-
- Pair<String, String> pair = TableDesc.parseResourcePath(cacheKey);
- String table = pair.getFirst();
- String prj = pair.getSecond();
-
- if (prj == null) {
- for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
- broadcaster.notifyProjectSchemaUpdate(p.getName());
- }
- } else {
- broadcaster.notifyProjectSchemaUpdate(prj);
- }
- }
- }
-
- private class SrcTableExtSyncListener extends Broadcaster.Listener {
- @Override
- public void onClearAll(Broadcaster broadcaster) throws IOException {
- clearCache();
- }
-
- @Override
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
- throws IOException {
- if (event == Event.DROP)
- srcTableExtMap.removeLocal(cacheKey);
- else
- reloadTableExtAt(TableExtDesc.concatRawResourcePath(cacheKey));
- }
- }
-
- private class DataModelSyncListener extends Broadcaster.Listener {
- @Override
- public void onClearAll(Broadcaster broadcaster) throws IOException {
- clearCache();
- }
-
- @Override
- public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
- for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
- reloadDataModelDescAt(DataModelDesc.concatResourcePath(model));
- }
- }
-
- @Override
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
- throws IOException {
- if (event == Event.DROP)
- dataModelDescMap.removeLocal(cacheKey);
- else
- reloadDataModelDescAt(DataModelDesc.concatResourcePath(cacheKey));
-
- for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(cacheKey)) {
- broadcaster.notifyProjectSchemaUpdate(prj.getName());
- }
- }
- }
-
- private class ExtFilterSyncListener extends Broadcaster.Listener {
- @Override
- public void onClearAll(Broadcaster broadcaster) throws IOException {
- clearCache();
- }
-
- @Override
- public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
- throws IOException {
- if (event == Event.DROP)
- extFilterMap.removeLocal(cacheKey);
- else
- reloadExtFilter(cacheKey);
- }
- }
-
- private void reloadAllTableExt() throws IOException {
- ResourceStore store = getStore();
- logger.debug("Reloading Table_exd info from folder "
- + store.getReadableResourcePath(ResourceStore.TABLE_EXD_RESOURCE_ROOT));
-
- srcTableExtMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_EXD_RESOURCE_ROOT,
- MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- reloadTableExtAt(path);
- }
-
- logger.debug("Loaded " + srcTableExtMap.size() + " SourceTable EXD(s)");
- }
-
- private TableExtDesc reloadTableExtAt(String path) throws IOException {
- ResourceStore store = getStore();
- String prj = TableExtDesc.parseResourcePath(path).getSecond();
-
- TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
-
- if (t == null) {
- return null;
- }
-
- // convert old tableExt json to new one
- if (t.getIdentity() == null) {
- t = convertOldTableExtToNewer(path);
- }
-
- t.init(prj);
-
- srcTableExtMap.putLocal(mapKey(t.getIdentity(), prj), t);
- return t;
- }
-
- private String mapKey(String identity, String prj) {
- return prj == null ? identity : identity + "--" + prj;
- }
-
- private TableExtDesc convertOldTableExtToNewer(String path) throws IOException {
- Map<String, String> attrs = Maps.newHashMap();
-
- ResourceStore store = getStore();
- RawResource res = store.getResource(path);
-
- InputStream is = res.inputStream;
-
- try {
- attrs.putAll(JsonUtil.readValue(is, HashMap.class));
- } finally {
- if (is != null)
- is.close();
- }
-
- String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY);
-
- // parse table identity from file name
- String file = path;
- if (file.indexOf("/") > -1) {
- file = file.substring(file.lastIndexOf("/") + 1);
- }
- String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase();
- TableExtDesc result = new TableExtDesc();
- result.setIdentity(tableIdentity);
- result.setUuid(UUID.randomUUID().toString());
- result.setLastModified(0);
- result.setCardinality(cardinality);
- return result;
- }
-
- private void reloadAllExternalFilter() throws IOException {
- ResourceStore store = getStore();
- logger.debug("Reloading ExternalFilter from folder "
- + store.getReadableResourcePath(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT));
-
- extFilterMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.EXTERNAL_FILTER_RESOURCE_ROOT,
- MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- reloadExternalFilterAt(path);
- }
-
- logger.debug("Loaded " + extFilterMap.size() + " ExternalFilter(s)");
- }
-
- private void reloadAllSourceTable() throws IOException {
- ResourceStore store = getStore();
- logger.debug("Reloading SourceTable from folder "
- + store.getReadableResourcePath(ResourceStore.TABLE_RESOURCE_ROOT));
-
- srcTableMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.TABLE_RESOURCE_ROOT,
- MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- reloadSourceTableAt(path);
- }
-
- logger.debug("Loaded " + srcTableMap.size() + " SourceTable(s)");
- }
-
- private TableDesc reloadSourceTableAt(String path) throws IOException {
- ResourceStore store = getStore();
- String prj = TableDesc.parseResourcePath(path).getSecond();
-
- TableDesc t = store.getResource(path, TableDesc.class, TABLE_SERIALIZER);
- if (t == null) {
- return null;
- }
- t.init(prj);
-
- srcTableMap.putLocal(mapKey(t.getIdentity(), prj), t);
-
- return t;
- }
-
- private ExternalFilterDesc reloadExternalFilterAt(String path) throws IOException {
- ResourceStore store = getStore();
- ExternalFilterDesc t = store.getResource(path, ExternalFilterDesc.class, EXTERNAL_FILTER_DESC_SERIALIZER);
- if (t == null) {
- return null;
- }
- extFilterMap.putLocal(t.getName(), t);
-
- return t;
- }
-
- public void reloadExtFilter(String extFilterName) throws IOException {
- reloadExternalFilterAt(ExternalFilterDesc.concatResourcePath(extFilterName));
- }
-
- public DataModelDesc getDataModelDesc(String name) {
- return dataModelDescMap.get(name);
- }
-
- public List<DataModelDesc> getModels() {
- return new ArrayList<>(dataModelDescMap.values());
- }
-
- public List<DataModelDesc> getModels(String projectName) {
- ProjectInstance projectInstance = ProjectManager.getInstance(config).getProject(projectName);
- ArrayList<DataModelDesc> ret = new ArrayList<>();
-
- if (projectInstance != null && projectInstance.getModels() != null) {
- for (String modelName : projectInstance.getModels()) {
- DataModelDesc model = getDataModelDesc(modelName);
- if (null != model) {
- ret.add(model);
- } else {
- logger.error("Failed to load model " + modelName);
- }
- }
- }
-
- return ret;
- }
-
- // within a project, find models that use the specified table
- public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException {
- List<String> models = new ArrayList<>();
- for (DataModelDesc modelDesc : getModels(project)) {
- if (modelDesc.containsTable(table))
- models.add(modelDesc.getName());
- }
- return models;
- }
-
- public boolean isTableInAnyModel(TableDesc table) {
- for (DataModelDesc modelDesc : getModels()) {
- if (modelDesc.containsTable(table))
- return true;
- }
- return false;
- }
-
- private void reloadAllDataModel() throws IOException {
- ResourceStore store = getStore();
- logger.debug("Reloading DataModel from folder "
- + store.getReadableResourcePath(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT));
-
- dataModelDescMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT,
- MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
-
- try {
- logger.info("Reloading data model at " + path);
- reloadDataModelDescAt(path);
- } catch (IllegalStateException e) {
- logger.error("Error to load DataModel at " + path, e);
- continue;
- }
- }
-
- logger.debug("Loaded " + dataModelDescMap.size() + " DataModel(s)");
- }
-
- public DataModelDesc reloadDataModelDescAt(String path) {
- ResourceStore store = getStore();
- try {
- DataModelDesc dataModelDesc = store.getResource(path, DataModelDesc.class, MODELDESC_SERIALIZER);
- String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName();
-
- if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels());
-
- dataModelDescMap.putLocal(dataModelDesc.getName(), dataModelDesc);
- return dataModelDesc;
- } catch (Exception e) {
- throw new IllegalStateException("Error to load " + path, e);
- }
- }
-
- // sync on update
- public DataModelDesc dropModel(DataModelDesc desc) throws IOException {
- logger.info("Dropping model '" + desc.getName() + "'");
- ResourceStore store = getStore();
- store.deleteResource(desc.getResourcePath());
- // delete model from project
- ProjectManager.getInstance(config).removeModelFromProjects(desc.getName());
- // clean model cache
- this.afterModelDropped(desc);
- return desc;
- }
-
- private void afterModelDropped(DataModelDesc desc) {
- removeModelCache(desc.getName());
- }
-
- public void removeModelCache(String modelName) {
- dataModelDescMap.remove(modelName);
- }
-
- public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException {
- String name = desc.getName();
- if (dataModelDescMap.containsKey(name))
- throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists");
-
- ProjectManager prjMgr = ProjectManager.getInstance(config);
- ProjectInstance prj = prjMgr.getProject(projectName);
- if (prj.containsModel(name))
- throw new IllegalStateException("project " + projectName + " already contains model " + name);
-
- try {
- // Temporarily register model under project, because we want to
- // update project formally after model is saved.
- prj.getModels().add(name);
-
- desc.setOwner(owner);
- desc = saveDataModelDesc(desc);
-
- } finally {
- prj.getModels().remove(name);
- }
-
- // now that model is saved, update project formally
- prjMgr.updateModelToProject(name, projectName);
-
- return desc;
- }
-
- public DataModelDesc updateDataModelDesc(DataModelDesc desc) throws IOException {
- String name = desc.getName();
- if (!dataModelDescMap.containsKey(name)) {
- throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist.");
- }
-
- return saveDataModelDesc(desc);
- }
-
- private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
-
- String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName();
-
- if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(prj), listDataModels());
-
- String path = dataModelDesc.getResourcePath();
- getStore().putResource(path, dataModelDesc, MODELDESC_SERIALIZER);
- dataModelDescMap.put(dataModelDesc.getName(), dataModelDesc);
-
- return dataModelDesc;
- }
-}
[03/11] kylin git commit: KYLIN-2917 make sure alias on OLAPTableScan
won't dup within a context
Posted by li...@apache.org.
KYLIN-2917 make sure alias on OLAPTableScan won't dup within a context
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d5bfcf5a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d5bfcf5a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d5bfcf5a
Branch: refs/heads/master
Commit: d5bfcf5a9e3cfe2546b73db9690d092d651082fe
Parents: 2415fd9
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 30 16:57:40 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 30 17:02:57 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/query/relnode/OLAPTableScan.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d5bfcf5a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index b739326..afdfde6 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -278,7 +278,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
}
private ColumnRowType buildColumnRowType() {
- this.alias = Integer.toHexString(System.identityHashCode(this));
+ this.alias = context.allTableScans.size() + "_" + Integer.toHexString(System.identityHashCode(this));
TableRef tableRef = TblColRef.tableForUnknownModel(this.alias, olapTable.getSourceTable());
List<TblColRef> columns = new ArrayList<TblColRef>();
[05/11] kylin git commit: KYLIN-2922 Support column used as dimension
and sum(column) at the same time
Posted by li...@apache.org.
KYLIN-2922 Support column used as dimension and sum(column) at the same time
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/92959476
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/92959476
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/92959476
Branch: refs/heads/master
Commit: 92959476bdd3e4775428a24f15fbc41f4ec2efd7
Parents: f45d813
Author: Li Yang <li...@apache.org>
Authored: Mon Oct 2 17:53:55 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 18:31:56 2017 +0800
----------------------------------------------------------------------
.../kylin/measure/basic/BasicMeasureType.java | 4 +-
.../kylin/measure/topn/TopNMeasureType.java | 6 ++-
.../apache/kylin/metadata/model/ColumnDesc.java | 4 +-
.../kylin/metadata/model/FunctionDesc.java | 6 +--
.../query/sql_verifyCount/query02.sql.expected | 2 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 4 +-
.../apache/kylin/query/schema/OLAPTable.java | 44 +++++++++++---------
7 files changed, 39 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index ed493a1..b0836df 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -26,7 +26,7 @@ import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
-@SuppressWarnings("rawtypes")
+@SuppressWarnings({ "rawtypes", "serial" })
public class BasicMeasureType extends MeasureType {
public static class Factory extends MeasureTypeFactory {
@@ -141,6 +141,6 @@ public class BasicMeasureType extends MeasureType {
@Override
public boolean needRewrite() {
- return !isSum();
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 9a04bdc..205538e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -380,10 +380,12 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>> {
literalTupleIdx[i] = tupleInfo.hasColumn(colRef) ? tupleInfo.getColumnIndex(colRef) : -1;
}
- // for TopN, the aggr must be SUM, so the number fill into the column position (without rewrite)
+ // for TopN, the aggr must be SUM
final int numericTupleIdx;
if (numericCol != null) {
- numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol) : -1;
+ FunctionDesc sumFunc = FunctionDesc.newInstance(FunctionDesc.FUNC_SUM, ParameterDesc.newInstance(numericCol), numericCol.getType().toString());
+ String sumFieldName = sumFunc.getRewriteFieldName();
+ numericTupleIdx = tupleInfo.hasField(sumFieldName) ? tupleInfo.getFieldIndex(sumFieldName) : -1;
} else {
FunctionDesc countFunction = FunctionDesc.newInstance(FunctionDesc.FUNC_COUNT, ParameterDesc.newInstance("1"), "bigint");
numericTupleIdx = tupleInfo.getFieldIndex(countFunction.getRewriteFieldName());
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 5848838..ecef1ba 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -115,8 +115,8 @@ public class ColumnDesc implements Serializable {
type = DataType.getType(datatype);
}
- public void setUpgradedType(String datatype) {
- this.upgradedType = DataType.getType(datatype);
+ public void setUpgradedType(DataType upgradedType) {
+ this.upgradedType = upgradedType;
}
public DataType getUpgradedType() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index e969be2..ce0b4c6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -136,9 +136,7 @@ public class FunctionDesc implements Serializable {
}
public String getRewriteFieldName() {
- if (isSum()) {
- return getParameter().getValue();
- } else if (isCount()) {
+ if (isCount()) {
return "_KY_" + "COUNT__"; // ignores parameter, count(*), count(1), count(col) are all the same
} else if (isCountDistinct()) {
return "_KY_" + getFullExpressionInAlphabetOrder().replaceAll("[(),. ]", "_");
@@ -148,7 +146,7 @@ public class FunctionDesc implements Serializable {
}
public DataType getRewriteFieldType() {
- if (isSum() || isMax() || isMin())
+ if (isMax() || isMin())
return parameter.getColRefs().get(0).getType();
else if (getMeasureType() instanceof BasicMeasureType)
return returnDataType;
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
index 6fbbabd..0044301 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
@@ -1,2 +1,2 @@
*
-18
+20
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 1540483..c6d40be 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -388,6 +388,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
return !this.context.hasPrecalculatedFields() || !RewriteImplementor.needRewrite(this.context);
}
+ @SuppressWarnings("deprecation")
private AggregateCall rewriteAggregateCall(AggregateCall aggCall, FunctionDesc func) {
// if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields, which do not exist at all
@@ -400,6 +401,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
String callName = getSqlFuncName(aggCall);
RelDataType fieldType = aggCall.getType();
SqlAggFunction newAgg = aggCall.getAggregation();
+
Map<String, Class<?>> udafMap = func.getMeasureType().getRewriteCalciteAggrFunctions();
if (func.isCount()) {
newAgg = SqlStdOperatorTable.SUM0;
@@ -423,8 +425,8 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
// rebuild aggregate call
- @SuppressWarnings("deprecation")
AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, callName);
+
return newAggCall;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/92959476/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 3401033..82fc8cb 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -63,7 +63,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
/**
*/
@@ -135,6 +134,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
return this.rowType;
}
+ @SuppressWarnings("deprecation")
private RelDataType deriveRowType(RelDataTypeFactory typeFactory) {
RelDataTypeFactory.FieldInfoBuilder fieldInfo = typeFactory.builder();
for (ColumnDesc column : sourceColumns) {
@@ -196,16 +196,19 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
//if exist sum(x), where x is integer/short/byte
//to avoid overflow we upgrade x's type to long
+ HashSet<ColumnDesc> upgraded = new HashSet<>();
+
//this includes checking two parts:
//1. sum measures in cubes:
- HashSet<ColumnDesc> upgradeCols = Sets.newHashSet();
for (MeasureDesc m : mgr.listEffectiveMeasures(olapSchema.getProjectName(), sourceTable.getIdentity())) {
if (m.getFunction().isSum()) {
FunctionDesc func = m.getFunction();
- if (func.getReturnDataType() != func.getRewriteFieldType() && //
- func.getReturnDataType().isBigInt() && //
- func.getRewriteFieldType().isIntegerFamily()) {
- upgradeCols.add(func.getParameter().getColRefs().get(0).getColumnDesc());
+ ColumnDesc col = func.getParameter().getColRefs().get(0).getColumnDesc();
+ if (col.getType() != func.getReturnDataType()) {
+ if (upgraded.contains(col) == false) {
+ upgradeColumnType(tableColumns, col, func.getReturnDataType());
+ upgraded.add(col);
+ }
}
}
}
@@ -217,23 +220,16 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
for (String metricColumn : model.getMetrics()) {
TblColRef col = model.findColumn(metricColumn);
if (col.getTable().equals(sourceTable.getIdentity()) && col.getType().isIntegerFamily()
- && !col.getType().isBigInt())
- upgradeCols.add(col.getColumnDesc());
+ && !col.getType().isBigInt()) {
+ if (upgraded.contains(col.getColumnDesc()) == false) {
+ upgradeColumnType(tableColumns, col.getColumnDesc(), DataType.getType("bigint"));
+ upgraded.add(col.getColumnDesc());
+ }
+ }
}
}
}
- for (ColumnDesc upgrade : upgradeCols) {
- int index = tableColumns.indexOf(upgrade);
- if (index < 0) {
- throw new IllegalStateException(
- "Metric column " + upgrade + " is not found in the the project's columns");
- }
- tableColumns.get(index).setUpgradedType("bigint");
- logger.info("To avoid overflow, upgraded {}'s type from {} to {}", tableColumns.get(index),
- tableColumns.get(index).getType(), tableColumns.get(index).getUpgradedType());
- }
-
Collections.sort(tableColumns, new Comparator<ColumnDesc>() {
@Override
public int compare(ColumnDesc o1, ColumnDesc o2) {
@@ -243,6 +239,16 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
}
+ private void upgradeColumnType(List<ColumnDesc> tableColumns, ColumnDesc upgrade, DataType newType) {
+ int index = tableColumns.indexOf(upgrade);
+ if (index < 0) {
+ throw new IllegalStateException("Metric column " + upgrade + " is not found in the the project's columns");
+ }
+ tableColumns.get(index).setUpgradedType(newType);
+ logger.info("To avoid overflow, upgraded {}'s type from {} to {}", tableColumns.get(index),
+ tableColumns.get(index).getType(), tableColumns.get(index).getUpgradedType());
+ }
+
@Override
public RelNode toRel(ToRelContext context, RelOptTable relOptTable) {
int fieldCount = relOptTable.getRowType().getFieldCount();
[07/11] kylin git commit: KYLIN-2921 Refactor DataModelDesc,
split DataModelManager
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 4346206..809e188 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -125,31 +125,6 @@
"column": "ACCOUNT_COUNTRY"
},
{
- "name": "DEAL_YEAR",
- "table": "TEST_KYLIN_FACT",
- "column": "DEAL_YEAR"
- },
- {
- "name": "SELLER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "SELLER_COUNTRY_ABBR"
- },
- {
- "name": "BUYER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "BUYER_COUNTRY_ABBR"
- },
- {
- "name": "SELLER_ID_AND_COUNTRY_NAME",
- "table": "TEST_KYLIN_FACT",
- "column": "SELLER_ID_AND_COUNTRY_NAME"
- },
- {
- "name": "BUYER_ID_AND_COUNTRY_NAME",
- "table": "TEST_KYLIN_FACT",
- "column": "BUYER_ID_AND_COUNTRY_NAME"
- },
- {
"name": "BUYER_COUNTRY_NAME",
"table": "BUYER_COUNTRY",
"column": "NAME"
@@ -343,17 +318,6 @@
},
"returntype": "raw"
}
- },
- {
- "name": "COMPUTED_COLUMN_MEASURE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "TEST_KYLIN_FACT.DEAL_AMOUNT"
- },
- "returntype": "decimal"
- }
}
],
"dictionaries": [
@@ -445,28 +409,8 @@
"encoding": "dict"
},
{
- "column": "TEST_KYLIN_FACT.DEAL_YEAR",
- "encoding": "int:2"
- },
- {
"column": "SELLER_COUNTRY.NAME",
"encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME",
- "encoding": "dict"
}
]
},
@@ -485,8 +429,7 @@
"ITEM_COUNT_SUM",
"GMV_SUM",
"GMV_MIN",
- "GMV_MAX",
- "COMPUTED_COLUMN_MEASURE"
+ "GMV_MAX"
]
}
]
@@ -533,8 +476,7 @@
"TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"
],
"select_rule": {
"hierarchy_dims": [
@@ -550,8 +492,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
]
],
"dim_cap": 3
@@ -579,11 +520,7 @@
"BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "BUYER_COUNTRY.NAME",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME"
+ "BUYER_COUNTRY.NAME"
],
"select_rule": {
"hierarchy_dims": [],
@@ -600,11 +537,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
],
[
"TEST_KYLIN_FACT.SELLER_ID",
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index 597f95f..20eef09 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -125,31 +125,6 @@
"column": "ACCOUNT_COUNTRY"
},
{
- "name": "DEAL_YEAR",
- "table": "TEST_KYLIN_FACT",
- "column": "DEAL_YEAR"
- },
- {
- "name": "SELLER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "SELLER_COUNTRY_ABBR"
- },
- {
- "name": "BUYER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "BUYER_COUNTRY_ABBR"
- },
- {
- "name": "SELLER_ID_AND_COUNTRY_NAME",
- "table": "TEST_KYLIN_FACT",
- "column": "SELLER_ID_AND_COUNTRY_NAME"
- },
- {
- "name": "BUYER_ID_AND_COUNTRY_NAME",
- "table": "TEST_KYLIN_FACT",
- "column": "BUYER_ID_AND_COUNTRY_NAME"
- },
- {
"name": "BUYER_COUNTRY_NAME",
"table": "BUYER_COUNTRY",
"column": "NAME"
@@ -354,17 +329,6 @@
},
"returntype": "percentile(100)"
}
- },
- {
- "name": "COMPUTED_COLUMN_MEASURE",
- "function": {
- "expression": "SUM",
- "parameter": {
- "type": "column",
- "value": "TEST_KYLIN_FACT.DEAL_AMOUNT"
- },
- "returntype": "decimal"
- }
}
],
"dictionaries": [
@@ -456,28 +420,8 @@
"encoding": "dict"
},
{
- "column": "TEST_KYLIN_FACT.DEAL_YEAR",
- "encoding": "int:2"
- },
- {
"column": "SELLER_COUNTRY.NAME",
"encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME",
- "encoding": "dict"
}
]
},
@@ -496,8 +440,7 @@
"ITEM_COUNT_SUM",
"GMV_SUM",
"GMV_MIN",
- "GMV_MAX",
- "COMPUTED_COLUMN_MEASURE"
+ "GMV_MAX"
]
}
]
@@ -545,8 +488,7 @@
"TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"
],
"select_rule": {
"hierarchy_dims": [
@@ -562,8 +504,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
]
],
"dim_cap": 5
@@ -591,11 +532,7 @@
"BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "BUYER_COUNTRY.NAME",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME"
+ "BUYER_COUNTRY.NAME"
],
"select_rule": {
"hierarchy_dims": [],
@@ -612,11 +549,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME",
- "TEST_KYLIN_FACT.BUYER_ID_AND_COUNTRY_NAME"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
],
[
"TEST_KYLIN_FACT.SELLER_ID",
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json b/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json
index 0e64166..9e2e002 100644
--- a/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json
+++ b/examples/test_case_data/localmeta/cube_desc/ut_inner_join_cube_partial.json
@@ -124,21 +124,6 @@
"column": "ACCOUNT_COUNTRY"
},
{
- "name": "DEAL_YEAR",
- "table": "TEST_KYLIN_FACT",
- "column": "DEAL_YEAR"
- },
- {
- "name": "SELLER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "SELLER_COUNTRY_ABBR"
- },
- {
- "name": "BUYER_COUNTRY_ABBR",
- "table": "TEST_KYLIN_FACT",
- "column": "BUYER_COUNTRY_ABBR"
- },
- {
"name": "BUYER_COUNTRY_NAME",
"table": "BUYER_COUNTRY",
"column": "NAME"
@@ -229,20 +214,8 @@
"encoding": "dict"
},
{
- "column": "TEST_KYLIN_FACT.DEAL_YEAR",
- "encoding": "int:2"
- },
- {
"column": "SELLER_COUNTRY.NAME",
"encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "encoding": "dict"
- },
- {
- "column": "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR",
- "encoding": "dict"
}
]
},
@@ -261,8 +234,7 @@
"TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
- "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME"
],
"select_rule": {
"hierarchy_dims": [
@@ -278,8 +250,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.DEAL_YEAR"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
]
],
"dim_cap": 3
@@ -307,9 +278,7 @@
"BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
"BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "BUYER_COUNTRY.NAME",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR"
+ "BUYER_COUNTRY.NAME"
],
"select_rule": {
"hierarchy_dims": [],
@@ -326,9 +295,7 @@
[
"TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"TEST_KYLIN_FACT.LSTG_SITE_ID",
- "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
- "TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR",
- "TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR"
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD"
],
[
"TEST_KYLIN_FACT.SELLER_ID",
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
index dbedc66..a3c3c05 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -121,55 +121,6 @@
}
}
],
- "computed_columns": [
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "columnName": "DEAL_AMOUNT",
- "expression": "PRICE * ITEM_COUNT",
- "datatype": "decimal",
- "comment": "deal amount of inner join model (with legacy expression format)"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "DEAL_YEAR",
- "expression": "year(TEST_KYLIN_FACT.CAL_DT)",
- "datatype": "integer",
- "comment": "the year of the deal"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "BUYER_ID_AND_COUNTRY_NAME",
- "expression": "CONCAT(BUYER_ACCOUNT.ACCOUNT_ID, BUYER_COUNTRY.NAME)",
- "datatype": "string",
- "comment": "synthetically concat buyer's account id and buyer country"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "SELLER_ID_AND_COUNTRY_NAME",
- "expression": "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)",
- "datatype": "string",
- "comment": "synthetically concat seller's account id and seller country"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "BUYER_COUNTRY_ABBR",
- "expression": "SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
- "datatype": "string",
- "comment": "first char of country of buyer account"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "SELLER_COUNTRY_ABBR",
- "expression": "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
- "datatype": "string",
- "comment": "first char of country of seller account"
- }
- ],
"dimensions": [
{
"table": "TEST_KYLIN_FACT",
@@ -182,12 +133,7 @@
"LEAF_CATEG_ID",
"SLR_SEGMENT_CD",
"SELLER_ID",
- "TEST_COUNT_DISTINCT_BITMAP",
- "DEAL_YEAR",
- "SELLER_COUNTRY_ABBR",
- "BUYER_COUNTRY_ABBR",
- "SELLER_ID_AND_COUNTRY_NAME",
- "BUYER_ID_AND_COUNTRY_NAME"
+ "TEST_COUNT_DISTINCT_BITMAP"
]
},
{
@@ -273,8 +219,7 @@
],
"metrics": [
"TEST_KYLIN_FACT.PRICE",
- "TEST_KYLIN_FACT.ITEM_COUNT",
- "TEST_KYLIN_FACT.DEAL_AMOUNT"
+ "TEST_KYLIN_FACT.ITEM_COUNT"
],
"last_modified": 1422435345352,
"filter_condition": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
index 7220b4d..2994984 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -121,55 +121,6 @@
}
}
],
- "computed_columns": [
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "columnName": "DEAL_AMOUNT",
- "expression": "PRICE * ITEM_COUNT",
- "datatype": "decimal",
- "comment": "deal amount of left join model (with legacy expression format)"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "DEAL_YEAR",
- "expression": "year(TEST_KYLIN_FACT.CAL_DT)",
- "datatype": "integer",
- "comment": "the year of the deal"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "BUYER_ID_AND_COUNTRY_NAME",
- "expression": "CONCAT(BUYER_ACCOUNT.ACCOUNT_ID, BUYER_COUNTRY.NAME)",
- "datatype": "string",
- "comment": "synthetically concat buyer's account id and buyer country"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "SELLER_ID_AND_COUNTRY_NAME",
- "expression": "CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)",
- "datatype": "string",
- "comment": "synthetically concat seller's account id and seller country"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "BUYER_COUNTRY_ABBR",
- "expression": "SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
- "datatype": "string",
- "comment": "first char of country of buyer account"
- },
- {
- "tableIdentity": "DEFAULT.TEST_KYLIN_FACT",
- "tableAlias": "TEST_KYLIN_FACT",
- "columnName": "SELLER_COUNTRY_ABBR",
- "expression": "SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1)",
- "datatype": "string",
- "comment": "first char of country of seller account"
- }
- ],
"dimensions": [
{
"table": "TEST_KYLIN_FACT",
@@ -182,12 +133,7 @@
"LEAF_CATEG_ID",
"SLR_SEGMENT_CD",
"SELLER_ID",
- "TEST_COUNT_DISTINCT_BITMAP",
- "DEAL_YEAR",
- "SELLER_COUNTRY_ABBR",
- "BUYER_COUNTRY_ABBR",
- "SELLER_ID_AND_COUNTRY_NAME",
- "BUYER_ID_AND_COUNTRY_NAME"
+ "TEST_COUNT_DISTINCT_BITMAP"
]
},
{
@@ -273,8 +219,7 @@
],
"metrics": [
"TEST_KYLIN_FACT.PRICE",
- "TEST_KYLIN_FACT.ITEM_COUNT",
- "TEST_KYLIN_FACT.DEAL_AMOUNT"
+ "TEST_KYLIN_FACT.ITEM_COUNT"
],
"last_modified": 1422435345352,
"filter_condition": null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index bda01ed..d703941 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
@@ -79,7 +79,7 @@ public class H2Database {
}
private void loadH2Table(String tableName) throws SQLException {
- MetadataManager metaMgr = MetadataManager.getInstance(config);
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
TableDesc tableDesc = metaMgr.getTableDesc(tableName.toUpperCase(), project);
File tempFile = null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
index 2fb7771..60633c9 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java
@@ -210,7 +210,6 @@ public class ITKylinQueryTest extends KylinTestBase {
} finally {
BackdoorToggles.cleanToggles();
-
}
}
@@ -261,11 +260,6 @@ public class ITKylinQueryTest extends KylinTestBase {
}
@Test
- public void testComputedColumnsQuery() throws Exception {
- execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_computedcolumn", null, true, CompareQueryBySuffix.INSTANCE);
- }
-
- @Test
public void testTopNQuery() throws Exception {
if ("left".equalsIgnoreCase(joinType)) {
this.execAndCompQuery(getQueryFolderPrefix() + "src/test/resources/query/sql_topn", null, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
index 8b75a28..384aa95 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
@@ -24,7 +24,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.kylin.common.util.HBaseMetadataTestCase;
import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.IReadableTable;
import org.apache.kylin.source.IReadableTable.TableReader;
@@ -55,7 +55,7 @@ public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
@Test
public void basicTest() throws Exception {
String tableName = "EDW.TEST_SITES";
- TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
+ TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
IReadableTable hiveTable = SourceFactory.createReadableTable(tableDesc);
String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
index 1bf3bfe..3869cb6 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java
@@ -27,8 +27,8 @@ import java.sql.SQLException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -63,7 +63,7 @@ public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase imple
String project = ProjectInstance.DEFAULT_PROJECT_NAME;
H2Database h2DB = new H2Database(h2Connection, config, project);
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
gen.generate();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
index 41a35fe..4a5bfe4 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java
@@ -24,8 +24,8 @@ import java.sql.SQLException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ISourceAware;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.H2Database;
import org.apache.kylin.source.datagen.ModelDataGenerator;
@@ -56,7 +56,7 @@ public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements
String project = ProjectInstance.DEFAULT_PROJECT_NAME;
H2Database h2DB = new H2Database(h2Connection, config, project);
- MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager mgr = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000);
gen.generate();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql
deleted file mode 100644
index 6aba6ce..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR in ('I', 'F')
-group by SELLER_COUNTRY.NAME, DEAL_YEAR
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare
deleted file mode 100644
index 3f21647..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query01.sql.compare
+++ /dev/null
@@ -1,14 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,0,1) in ('I', 'F')
-group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql
deleted file mode 100644
index 3feafee..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and TEST_KYLIN_FACT.SELLER_COUNTRY_ABBR in ('T', 'R') and TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR in ('T', 'R')
-group by SELLER_COUNTRY.NAME, DEAL_YEAR
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare
deleted file mode 100644
index 7243d7e..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query02.sql.compare
+++ /dev/null
@@ -1,21 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR(SELLER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R') and SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R')
-group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
deleted file mode 100644
index 06eabeb..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(DEAL_AMOUNT) as deal_amount, SELLER_COUNTRY.NAME, DEAL_YEAR as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and "TEST_KYLIN_FACT"."SELLER_COUNTRY_ABBR" in ('T', 'R') and TEST_KYLIN_FACT.BUYER_COUNTRY_ABBR in ('T', 'R')
-group by SELLER_COUNTRY.NAME, DEAL_YEAR
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
deleted file mode 100644
index 160375c..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query03.sql.compare
+++ /dev/null
@@ -1,21 +0,0 @@
-SELECT
-
-count(*) as cnt, sum(price) as sum_price, sum(TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as deal_amount, SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT) as deal_year
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-where SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL=1 and SUBSTR("SELLER_ACCOUNT"."ACCOUNT_COUNTRY",1,1) in ('T', 'R') and SUBSTR(BUYER_ACCOUNT.ACCOUNT_COUNTRY,1,1) in ('T', 'R')
-group by SELLER_COUNTRY.NAME, year(TEST_KYLIN_FACT.CAL_DT)
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
deleted file mode 100644
index 75af19b..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-select count(*) as a , sum (TEST_KYLIN_FACT.DEAL_AMOUNT) as b , TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME as c
-
-
-
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-group by TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME
-
- order by b
-
-
-
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare b/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
deleted file mode 100644
index 66ffc07..0000000
--- a/kylin-it/src/test/resources/query/sql_computedcolumn/query04.sql.compare
+++ /dev/null
@@ -1,26 +0,0 @@
-select count(*) as a, sum (TEST_KYLIN_FACT.PRICE * TEST_KYLIN_FACT.ITEM_COUNT) as b, CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME) as c
-
-
-
-FROM TEST_KYLIN_FACT as TEST_KYLIN_FACT
-INNER JOIN TEST_ORDER as TEST_ORDER
-ON TEST_KYLIN_FACT.ORDER_ID = TEST_ORDER.ORDER_ID
-INNER JOIN TEST_ACCOUNT as BUYER_ACCOUNT
-ON TEST_ORDER.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_ACCOUNT as SELLER_ACCOUNT
-ON TEST_KYLIN_FACT.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID
-INNER JOIN TEST_CATEGORY_GROUPINGS as TEST_CATEGORY_GROUPINGS
-ON TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID
-INNER JOIN TEST_COUNTRY as BUYER_COUNTRY
-ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY
-INNER JOIN TEST_COUNTRY as SELLER_COUNTRY
-ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY
-
-
-
-
-
-
-
- group by CONCAT(SELLER_ACCOUNT.ACCOUNT_ID, SELLER_COUNTRY.NAME)
- order by b
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected
index d2f0772..86242f2 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query01.sql.expected
@@ -1,2 +1,2 @@
*
-17
+11
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
index e530f14..6fbbabd 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query02.sql.expected
@@ -1,2 +1,2 @@
*
-24
+18
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected
index 3097aec..093e327 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query03.sql.expected
@@ -1,2 +1,2 @@
10
-19
+13
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected
index 0b28614..71011db 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query04.sql.expected
@@ -1,2 +1,2 @@
100
-17
+11
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected
index d2f0772..86242f2 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query10.sql.expected
@@ -1,2 +1,2 @@
*
-17
+11
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected
index a21e30c..b5fa014 100644
--- a/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected
+++ b/kylin-it/src/test/resources/query/sql_verifyCount/query11.sql.expected
@@ -1,2 +1,2 @@
10
-17
+11
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/query/.settings/org.eclipse.core.resources.prefs
----------------------------------------------------------------------
diff --git a/query/.settings/org.eclipse.core.resources.prefs b/query/.settings/org.eclipse.core.resources.prefs
index 839d647..29abf99 100644
--- a/query/.settings/org.eclipse.core.resources.prefs
+++ b/query/.settings/org.eclipse.core.resources.prefs
@@ -2,4 +2,5 @@ eclipse.preferences.version=1
encoding//src/main/java=UTF-8
encoding//src/main/resources=UTF-8
encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
encoding/<project>=UTF-8
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/query/src/main/java/org/apache/kylin/query/QueryConnection.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/QueryConnection.java b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
index f659533..21e3b1b 100644
--- a/query/src/main/java/org/apache/kylin/query/QueryConnection.java
+++ b/query/src/main/java/org/apache/kylin/query/QueryConnection.java
@@ -26,12 +26,10 @@ import java.util.Properties;
import org.apache.calcite.jdbc.Driver;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.log4j.Logger;
public class QueryConnection {
- private static final Logger logger = Logger.getLogger(QueryConnection.class);
+
private static Boolean isRegister = false;
public static Connection getConnection(String project) throws SQLException {
@@ -39,8 +37,7 @@ public class QueryConnection {
DriverManager.registerDriver(new Driver());
isRegister = true;
}
- File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.getNormalizedProjectName(project),
- KylinConfig.getInstanceFromEnv());
+ File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project, KylinConfig.getInstanceFromEnv());
Properties info = new Properties();
info.put("model", olapTmp.getAbsolutePath());
return DriverManager.getConnection("jdbc:calcite:", info);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
index 5387a74..fd1972a 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java
@@ -27,9 +27,8 @@ import org.apache.calcite.schema.impl.AbstractSchema;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.StorageURL;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
/**
@@ -56,7 +55,7 @@ public class OLAPSchema extends AbstractSchema {
}
public OLAPSchema(String project, String schemaName, boolean exposeMore) {
- this.projectName = ProjectInstance.getNormalizedProjectName(project);
+ this.projectName = project;
this.schemaName = schemaName;
this.exposeMore = exposeMore;
init();
@@ -115,8 +114,8 @@ public class OLAPSchema extends AbstractSchema {
return starSchemaPassword;
}
- public MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
+ public DataModelManager getMetadataManager() {
+ return DataModelManager.getInstance(config);
}
public KylinConfig getConfig() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
index 2e9c951..3c57edb 100644
--- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
+++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchemaFactory.java
@@ -37,7 +37,6 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.DatabaseDesc;
import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +58,6 @@ public class OLAPSchemaFactory implements SchemaFactory {
}
public static File createTempOLAPJson(String project, KylinConfig config) {
- project = ProjectInstance.getNormalizedProjectName(project);
Collection<TableDesc> tables = ProjectManager.getInstance(config).listExposedTables(project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
index cf9f3e3..6ce4c8e 100644
--- a/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/security/QueryInterceptorUtil.java
@@ -19,29 +19,10 @@
package org.apache.kylin.query.security;
import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.relnode.OLAPTableScan;
-
-import com.google.common.base.Preconditions;
public class QueryInterceptorUtil {
private static List<QueryInterceptor> queryInterceptors = new ArrayList<>();
@@ -66,105 +47,4 @@ public class QueryInterceptorUtil {
return queryInterceptors;
}
- public static Set<String> getAllColsWithTblAndSchema(String project, List<OLAPContext> contexts) {
- // all columns with table and DB. Like DB.TABLE.COLUMN
- Set<String> allColWithTblAndSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-
- for (OLAPContext context : contexts) {
- for (TblColRef tblColRef : context.allColumns) {
- ColumnDesc columnDesc = tblColRef.getColumnDesc();
- //computed column
- if (columnDesc.isComputedColumnn()) {
- allColWithTblAndSchema.addAll(getCCUsedCols(project, columnDesc));
- }
- //normal column
- allColWithTblAndSchema.add(tblColRef.getColumWithTableAndSchema());
- }
- }
- return allColWithTblAndSchema;
- }
-
- private static Set<String> getCCUsedCols(String project, ColumnDesc columnDesc) {
- Set<String> usedCols = new HashSet<>();
- Map<String, String> aliasTableMap = getAliasTableMap(project, columnDesc.getName());
- Preconditions.checkState(aliasTableMap.size() > 0, "can not find cc:" + columnDesc.getName() + "'s table alias");
-
- List<Pair<String, String>> colsWithAlias = ExprIdentifierFinder.getExprIdentifiers(columnDesc.getComputedColumnExpr());
- for (Pair<String, String> cols : colsWithAlias) {
- String tableIdentifier = aliasTableMap.get(cols.getFirst());
- usedCols.add(tableIdentifier + "." + cols.getSecond());
- }
- //Preconditions.checkState(usedCols.size() > 0, "can not find cc:" + columnDesc.getName() + "'s used cols");
- return usedCols;
- }
-
- private static Map<String, String> getAliasTableMap(String project, String ccName) {
- DataModelDesc model = getModel(project, ccName);
- Map<String, String> tableWithAlias = new HashMap<>();
- for (String alias : model.getAliasMap().keySet()) {
- String tableName = model.getAliasMap().get(alias).getTableDesc().getIdentity();
- tableWithAlias.put(alias, tableName);
- }
- return tableWithAlias;
- }
-
- private static DataModelDesc getModel(String project, String ccName) {
- List<DataModelDesc> models = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getModels(project);
- for (DataModelDesc model : models) {
- Set<String> computedColumnNames = model.getComputedColumnNames();
- if (computedColumnNames.contains(ccName)) {
- return model;
- }
- }
- return null;
- }
-
- static class ExprIdentifierFinder extends SqlBasicVisitor<SqlNode> {
- List<Pair<String, String>> columnWithTableAlias;
-
- ExprIdentifierFinder() {
- this.columnWithTableAlias = new ArrayList<>();
- }
-
- List<Pair<String, String>> getIdentifiers() {
- return columnWithTableAlias;
- }
-
- static List<Pair<String, String>> getExprIdentifiers(String expr) {
- SqlNode exprNode = CalciteParser.getExpNode(expr);
- ExprIdentifierFinder id = new ExprIdentifierFinder();
- exprNode.accept(id);
- return id.getIdentifiers();
- }
-
- @Override
- public SqlNode visit(SqlCall call) {
- for (SqlNode operand : call.getOperandList()) {
- if (operand != null) {
- operand.accept(this);
- }
- }
- return null;
- }
-
- @Override
- public SqlNode visit(SqlIdentifier id) {
- //Preconditions.checkState(id.names.size() == 2, "error when get identifier in cc's expr");
- if (id.names.size() == 2) {
- columnWithTableAlias.add(Pair.newPair(id.names.get(0), id.names.get(1)));
- }
- return null;
- }
- }
-
- public static Set<String> getAllTblsWithSchema(List<OLAPContext> contexts) {
- // all tables with DB, Like DB.TABLE
- Set<String> tableWithSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
- for (OLAPContext context : contexts) {
- for (OLAPTableScan tableScan : context.allTableScans) {
- tableWithSchema.add(tableScan.getTableRef().getTableIdentity());
- }
- }
- return tableWithSchema;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 0dc9dfa..afa914c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -26,8 +26,8 @@ import java.util.UUID;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
@@ -161,7 +161,7 @@ public class ModelController extends BasicController {
@RequestMapping(value = "/{modelName}", method = { RequestMethod.DELETE }, produces = { "application/json" })
@ResponseBody
public void deleteModel(@PathVariable String modelName) {
- DataModelDesc desc = modelService.getMetadataManager().getDataModelDesc(modelName);
+ DataModelDesc desc = modelService.getDataModelManager().getDataModelDesc(modelName);
if (null == desc) {
throw new NotFoundException("Data Model with name " + modelName + " not found..");
}
@@ -177,7 +177,7 @@ public class ModelController extends BasicController {
@ResponseBody
public ModelRequest cloneModel(@PathVariable String modelName, @RequestBody ModelRequest modelRequest) {
String project = modelRequest.getProject();
- MetadataManager metaManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager metaManager = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc modelDesc = metaManager.getDataModelDesc(modelName);
String newModelName = modelRequest.getModelName();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
index e6f337f..f2b21ab 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
@@ -21,8 +21,8 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
@@ -48,7 +48,7 @@ public class ModelDescController extends BasicController {
@RequestMapping(value = "/{model_name}", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public DataModelDesc getModel(@PathVariable String model_name) {
- MetadataManager metaManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelManager metaManager = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());
DataModelDesc modeDesc = metaManager.getDataModelDesc(model_name);
return modeDesc;
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
index 875e978..72ae1ce 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/HybridCubeCLI.java
@@ -33,8 +33,8 @@ import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.RealizationType;
@@ -70,7 +70,7 @@ public class HybridCubeCLI extends AbstractApplication {
private KylinConfig kylinConfig;
private CubeManager cubeManager;
private HybridManager hybridManager;
- private MetadataManager metadataManager;
+ private DataModelManager metadataManager;
private ResourceStore store;
public HybridCubeCLI() {
@@ -85,7 +85,7 @@ public class HybridCubeCLI extends AbstractApplication {
this.store = ResourceStore.getStore(kylinConfig);
this.cubeManager = CubeManager.getInstance(kylinConfig);
this.hybridManager = HybridManager.getInstance(kylinConfig);
- this.metadataManager = MetadataManager.getInstance(kylinConfig);
+ this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
public static void main(String[] args) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java b/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java
index 59c9090..b1049b3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/DataModelDescResponse.java
@@ -58,6 +58,5 @@ public class DataModelDescResponse extends DataModelDesc {
if (dataModelDesc.getPartitionDesc() != null)
setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc()));
setCapacity(dataModelDesc.getCapacity());
- setComputedColumnDescs(dataModelDesc.getComputedColumnDescs());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/security/TableInterceptor.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/TableInterceptor.java b/server-base/src/main/java/org/apache/kylin/rest/security/TableInterceptor.java
index e87a630..e129a18 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/security/TableInterceptor.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/security/TableInterceptor.java
@@ -20,12 +20,13 @@ package org.apache.kylin.rest.security;
import java.util.List;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.acl.TableACLManager;
import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.relnode.OLAPTableScan;
import org.apache.kylin.query.security.QueryInterceptor;
-import org.apache.kylin.query.security.QueryInterceptorUtil;
public class TableInterceptor extends QueryInterceptor {
@@ -36,7 +37,7 @@ public class TableInterceptor extends QueryInterceptor {
@Override
public Set<String> getQueryIdentifiers(List<OLAPContext> contexts) {
- return QueryInterceptorUtil.getAllTblsWithSchema(contexts);
+ return getAllTblsWithSchema(contexts);
}
@Override
@@ -54,4 +55,16 @@ public class TableInterceptor extends QueryInterceptor {
protected String getIdentifierType() {
return "table";
}
+
+ protected Set<String> getAllTblsWithSchema(List<OLAPContext> contexts) {
+ // all tables with DB, Like DB.TABLE
+ Set<String> tableWithSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ for (OLAPContext context : contexts) {
+ for (OLAPTableScan tableScan : context.allTableScans) {
+ tableWithSchema.add(tableScan.getTableRef().getTableIdentity());
+ }
+ }
+ return tableWithSchema;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index f662042..0e9ee7a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -24,10 +24,11 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.acl.TableACLManager;
import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.metadata.draft.DraftManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.streaming.StreamingManager;
import org.apache.kylin.source.kafka.KafkaConfigManager;
@@ -45,8 +46,12 @@ public abstract class BasicService {
return kylinConfig;
}
- public MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(getConfig());
+ public TableMetadataManager getTableManager() {
+ return TableMetadataManager.getInstance(getConfig());
+ }
+
+ public DataModelManager getDataModelManager() {
+ return DataModelManager.getInstance(getConfig());
}
public CubeManager getCubeManager() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index f19290f..ff6773a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -25,7 +25,6 @@ import javax.sql.DataSource;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.QueryDataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,7 +143,6 @@ public class CacheService extends BasicService implements InitializingBean {
if (StringUtils.isEmpty(project))
throw new IllegalArgumentException("removeOLAPDataSource: project name not given");
- project = ProjectInstance.getNormalizedProjectName(project);
queryDataSource.removeCache(project);
}
@@ -156,9 +154,6 @@ public class CacheService extends BasicService implements InitializingBean {
@Deprecated
public DataSource getOLAPDataSource(String project) {
-
- project = ProjectInstance.getNormalizedProjectName(project);
-
DataSource ret = queryDataSource.get(project, getConfig());
return ret;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 28859b6..a18aaf3 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -621,7 +621,7 @@ public class CubeService extends BasicService implements InitializingBean {
}
if (!isDraft) {
- DataModelDesc modelDesc = modelService.getMetadataManager().getDataModelDesc(desc.getModelName());
+ DataModelDesc modelDesc = modelService.getDataModelManager().getDataModelDesc(desc.getModelName());
if (modelDesc == null) {
throw new BadRequestException(String.format(msg.getMODEL_NOT_FOUND(), desc.getModelName()));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
index 90dac27..5bd6a81 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
@@ -40,25 +40,25 @@ public class ExtFilterService extends BasicService {
public void saveExternalFilter(ExternalFilterDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
- if (getMetadataManager().getExtFilterDesc(desc.getName()) != null) {
+ if (getTableManager().getExtFilterDesc(desc.getName()) != null) {
throw new BadRequestException(String.format(msg.getFILTER_ALREADY_EXIST(), desc.getName()));
}
- getMetadataManager().saveExternalFilter(desc);
+ getTableManager().saveExternalFilter(desc);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void updateExternalFilter(ExternalFilterDesc desc) throws IOException {
Message msg = MsgPicker.getMsg();
- if (getMetadataManager().getExtFilterDesc(desc.getName()) == null) {
+ if (getTableManager().getExtFilterDesc(desc.getName()) == null) {
throw new BadRequestException(String.format(msg.getFILTER_NOT_FOUND(), desc.getName()));
}
- getMetadataManager().saveExternalFilter(desc);
+ getTableManager().saveExternalFilter(desc);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void removeExternalFilter(String name) throws IOException {
- getMetadataManager().removeExternalFilter(name);
+ getTableManager().removeExternalFilter(name);
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -73,7 +73,7 @@ public class ExtFilterService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public List<ExternalFilterDesc> listAllExternalFilters() {
- return getMetadataManager().listAllExternalFilters();
+ return getTableManager().listAllExternalFilters();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
index cad39b4..912923a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HybridService.java
@@ -130,7 +130,7 @@ public class HybridService extends BasicService {
}
}
- DataModelDesc model = (null != modelName) ? getMetadataManager().getDataModelDesc(modelName) : null;
+ DataModelDesc model = (null != modelName) ? getDataModelManager().getDataModelDesc(modelName) : null;
if (StringUtils.isEmpty(modelName)) {
return hybridsInProject;
} else if (model == null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 05cf942..b099396 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -26,21 +26,13 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.HiveCmdBuilder;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.metadata.draft.Draft;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.JoinsTree;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
-import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -87,7 +79,7 @@ public class ModelService extends BasicService {
if (StringUtils.isEmpty(modelName) || !StringUtils.containsOnly(modelName, VALID_MODELNAME)) {
return false;
}
- for (DataModelDesc model : getMetadataManager().getModels()) {
+ for (DataModelDesc model : getDataModelManager().getModels()) {
if (modelName.equalsIgnoreCase(model.getName())) {
return false;
}
@@ -102,10 +94,10 @@ public class ModelService extends BasicService {
if (null == project) {
aclEvaluate.checkIsGlobalAdmin();
- models = getMetadataManager().getModels();
+ models = getDataModelManager().getModels();
} else {
aclEvaluate.hasProjectReadPermission(project);
- models = getMetadataManager().getModels(projectName);
+ models = getDataModelManager().getModels(projectName);
}
List<DataModelDesc> filterModels = new ArrayList<DataModelDesc>();
@@ -142,12 +134,12 @@ public class ModelService extends BasicService {
aclEvaluate.hasProjectWritePermission(getProjectManager().getProject(projectName));
Message msg = MsgPicker.getMsg();
- if (getMetadataManager().getDataModelDesc(desc.getName()) != null) {
+ if (getDataModelManager().getDataModelDesc(desc.getName()) != null) {
throw new BadRequestException(String.format(msg.getDUPLICATE_MODEL_NAME(), desc.getName()));
}
DataModelDesc createdDesc = null;
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
- createdDesc = getMetadataManager().createDataModelDesc(desc, projectName, owner);
+ createdDesc = getDataModelManager().createDataModelDesc(desc, projectName, owner);
if (!desc.isDraft()) {
accessService.init(createdDesc, AclPermission.ADMINISTRATION);
@@ -159,7 +151,7 @@ public class ModelService extends BasicService {
public DataModelDesc updateModelAndDesc(String project, DataModelDesc desc) throws IOException {
aclEvaluate.checkProjectWritePermission(project);
- getMetadataManager().updateDataModelDesc(desc);
+ getDataModelManager().updateDataModelDesc(desc);
return desc;
}
@@ -174,21 +166,21 @@ public class ModelService extends BasicService {
}
}
- getMetadataManager().dropModel(desc);
+ getDataModelManager().dropModel(desc);
accessService.clean(desc, true);
}
public boolean isTableInAnyModel(TableDesc table) {
- return getMetadataManager().isTableInAnyModel(table);
+ return getDataModelManager().isTableInAnyModel(table);
}
public boolean isTableInModel(TableDesc table, String project) throws IOException {
- return getMetadataManager().getModelsUsingTable(table, project).size() > 0;
+ return getDataModelManager().getModelsUsingTable(table, project).size() > 0;
}
public List<String> getModelsUsingTable(TableDesc table, String project) throws IOException {
- return getMetadataManager().getModelsUsingTable(table, project);
+ return getDataModelManager().getModelsUsingTable(table, project);
}
public Map<TblColRef, Set<CubeInstance>> getUsedDimCols(String modelName, String project) {
@@ -227,88 +219,6 @@ public class ModelService extends BasicService {
return ret;
}
- /**
- * check if the computed column expressions are valid ( in hive)
- */
- public boolean checkCCExpression(final DataModelDesc dataModelDesc, String project) throws IOException {
-
- dataModelDesc.setDraft(false);
- if (dataModelDesc.getUuid() == null)
- dataModelDesc.updateRandomUuid();
-
- dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(project),
- getMetadataManager().listDataModels());
-
- for (ComputedColumnDesc cc : dataModelDesc.getComputedColumnDescs()) {
-
- //check by calcite parser
- cc.simpleParserCheck(cc.getExpression(), dataModelDesc.getAliasMap().keySet());
-
- //check by hive cli, this could be slow
- StringBuilder sb = new StringBuilder();
- sb.append("select ");
- sb.append(cc.getExpression());
- sb.append(" ");
- JoinedFlatTable.appendJoinStatement(new IJoinedFlatTableDesc() {
- @Override
- public String getTableName() {
- return null;
- }
-
- @Override
- public DataModelDesc getDataModel() {
- return dataModelDesc;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return null;
- }
-
- @Override
- public int getColumnIndex(TblColRef colRef) {
- return 0;
- }
-
- @Override
- public SegmentRange getSegRange() {
- return null;
- }
-
- @Override
- public TblColRef getDistributedBy() {
- return null;
- }
-
- @Override
- public TblColRef getClusterBy() {
- return null;
- }
-
- @Override
- public ISegment getSegment() {
- return null;
- }
- }, sb, false);
- sb.append(" limit 0");
-
- final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
- hiveCmdBuilder.addStatement(sb.toString());
-
- long ts = System.currentTimeMillis();
- Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor()
- .execute(hiveCmdBuilder.toString());
- logger.debug("Spent " + (System.currentTimeMillis() - ts)
- + " ms to execute the hive command to validate computed column expression: " + cc.getExpression());
- if (response.getFirst() != 0) {
- throw new IllegalArgumentException("The expression " + cc.getExpression()
- + " failed syntax check with output message: " + response.getSecond());
- }
- }
-
- return true;
- }
-
private List<String> getModelCols(DataModelDesc model) {
List<String> dimCols = new ArrayList<String>();
@@ -377,8 +287,8 @@ public class ModelService extends BasicService {
StringBuilder checkRet = new StringBuilder();
if (cubes != null && cubes.size() != 0) {
- dataModelDesc.init(getConfig(), getMetadataManager().getAllTablesMap(project),
- getMetadataManager().listDataModels());
+ dataModelDesc.init(getConfig(), getTableManager().getAllTablesMap(project),
+ getDataModelManager().listDataModels());
List<String> curModelDims = getModelCols(dataModelDesc);
List<String> curModelMeasures = getModelMeasures(dataModelDesc);
@@ -475,7 +385,7 @@ public class ModelService extends BasicService {
aclEvaluate.hasProjectReadPermission(project);
}
- return getMetadataManager().getDataModelDesc(modelName);
+ return getDataModelManager().getDataModelDesc(modelName);
}
public Draft getModelDraft(String modelName, String projectName) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
index 71c29c6..965a995 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableSchemaUpdateChecker.java
@@ -28,7 +28,7 @@ import javax.annotation.Nullable;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -41,7 +41,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
public class TableSchemaUpdateChecker {
- private final MetadataManager metadataManager;
+ private final TableMetadataManager metadataManager;
private final CubeManager cubeManager;
static class CheckResult {
@@ -81,7 +81,7 @@ public class TableSchemaUpdateChecker {
}
}
- TableSchemaUpdateChecker(MetadataManager metadataManager, CubeManager cubeManager) {
+ TableSchemaUpdateChecker(TableMetadataManager metadataManager, CubeManager cubeManager) {
this.metadataManager = checkNotNull(metadataManager, "metadataManager is null");
this.cubeManager = checkNotNull(cubeManager, "cubeManager is null");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index a7c851c..ffb71a6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -38,7 +38,7 @@ import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -105,7 +105,7 @@ public class TableService extends BasicService {
public TableDesc getTableDescByName(String tableName, boolean withExt, String prj) {
aclEvaluate.checkProjectReadPermission(prj);
- TableDesc table = getMetadataManager().getTableDesc(tableName, prj);
+ TableDesc table = getTableManager().getTableDesc(tableName, prj);
if (withExt) {
aclEvaluate.checkProjectWritePermission(prj);
table = cloneTableDesc(table, prj);
@@ -137,8 +137,8 @@ public class TableService extends BasicService {
}
// do schema check
- MetadataManager metaMgr = MetadataManager.getInstance(getConfig());
- CubeManager cubeMgr = CubeManager.getInstance(getConfig());
+ TableMetadataManager metaMgr = getTableManager();
+ CubeManager cubeMgr = getCubeManager();
TableSchemaUpdateChecker checker = new TableSchemaUpdateChecker(metaMgr, cubeMgr);
for (Pair<TableDesc, TableExtDesc> pair : allMeta) {
TableDesc tableDesc = pair.getFirst();
@@ -247,7 +247,7 @@ public class TableService extends BasicService {
int tableType = 0;
tableName = normalizeHiveTableName(tableName);
- TableDesc desc = getMetadataManager().getTableDesc(tableName, project);
+ TableDesc desc = getTableManager().getTableDesc(tableName, project);
// unload of legacy global table is not supported for now
if (desc == null || desc.getProject() == null) {
@@ -266,7 +266,7 @@ public class TableService extends BasicService {
}
// it is a project local table, ready to remove since no model is using it within the project
- MetadataManager metaMgr = MetadataManager.getInstance(getConfig());
+ TableMetadataManager metaMgr = getTableManager();
metaMgr.removeTableExt(tableName, project);
metaMgr.removeSourceTable(tableName, project);
@@ -297,7 +297,7 @@ public class TableService extends BasicService {
public void addStreamingTable(TableDesc desc, String project) throws IOException {
aclEvaluate.checkProjectAdminPermission(project);
desc.setUuid(UUID.randomUUID().toString());
- getMetadataManager().saveSourceTable(desc, project);
+ getTableManager().saveSourceTable(desc, project);
syncTableToProject(new String[] { desc.getIdentity() }, project);
}
@@ -323,7 +323,7 @@ public class TableService extends BasicService {
}
private TableDescResponse cloneTableDesc(TableDesc table, String prj) {
- TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity(), prj);
+ TableExtDesc tableExtDesc = getTableManager().getTableExt(table.getIdentity(), prj);
// Clone TableDesc
TableDescResponse rtableDesc = new TableDescResponse(table);
@@ -362,7 +362,7 @@ public class TableService extends BasicService {
}
public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception {
- MetadataManager metaMgr = getMetadataManager();
+ TableMetadataManager metaMgr = getTableManager();
ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
for (String table : tables) {
TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj);
@@ -384,8 +384,8 @@ public class TableService extends BasicService {
Message msg = MsgPicker.getMsg();
tableName = normalizeHiveTableName(tableName);
- TableDesc table = getMetadataManager().getTableDesc(tableName, prj);
- final TableExtDesc tableExt = getMetadataManager().getTableExt(tableName, prj);
+ TableDesc table = getTableManager().getTableDesc(tableName, prj);
+ final TableExtDesc tableExt = getTableManager().getTableExt(tableName, prj);
if (table == null) {
BadRequestException e = new BadRequestException(String.format(msg.getTABLE_DESC_NOT_FOUND(), tableName));
logger.error("Cannot find table descriptor " + tableName, e);
@@ -416,7 +416,7 @@ public class TableService extends BasicService {
step2.setParam("segmentId", tableName);
job.addTask(step2);
tableExt.setJodID(job.getId());
- getMetadataManager().saveTableExt(tableExt, prj);
+ getTableManager().saveTableExt(tableExt, prj);
getExecutableManager().addJob(job);
}
[02/11] kylin git commit: minor,
fix NPE in ACL ValidateUtil getALlUsers
Posted by li...@apache.org.
minor, fix NPE in ACL ValidateUtil getALlUsers
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2415fd9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2415fd9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2415fd9e
Branch: refs/heads/master
Commit: 2415fd9e65b7bf483c375eda7d074c6804edb60e
Parents: bd7cb9c
Author: Billy Liu <bi...@apache.org>
Authored: Sat Sep 30 15:04:19 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 30 16:11:29 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/rest/util/ValidateUtil.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2415fd9e/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
index abb326f..7e513ae 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/ValidateUtil.java
@@ -122,8 +122,10 @@ public class ValidateUtil {
ProjectInstance prj = projectService.getProjectManager().getProject(project);
AclEntity ae = accessService.getAclEntity("ProjectInstance", prj.getUuid());
Acl acl = accessService.getAcl(ae);
- for (AccessControlEntry ace : acl.getEntries()) {
- allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+ if (acl != null && acl.getEntries() != null) {
+ for (AccessControlEntry ace : acl.getEntries()) {
+ allUsers.add(((PrincipalSid) ace.getSid()).getPrincipal());
+ }
}
return allUsers;
}
[04/11] kylin git commit: KYLIN-2917 bug fix
Posted by li...@apache.org.
KYLIN-2917 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1f6e04c8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1f6e04c8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1f6e04c8
Branch: refs/heads/master
Commit: 1f6e04c80d721ce11e415f92e9df1c4694d56bde
Parents: d5bfcf5
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 1 09:07:50 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 1 09:11:10 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/query/relnode/OLAPTableScan.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1f6e04c8/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
index afdfde6..142557c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java
@@ -216,9 +216,9 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel {
implementor.allocateContext();
}
- columnRowType = buildColumnRowType();
context = implementor.getContext();
context.allTableScans.add(this);
+ columnRowType = buildColumnRowType();
if (context.olapSchema == null) {
OLAPSchema schema = olapTable.getSchema();
[11/11] kylin git commit: Merge commit
'c04be637d2991afb3a97eac59c333e4ce28f5a43'
Posted by li...@apache.org.
Merge commit 'c04be637d2991afb3a97eac59c333e4ce28f5a43'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2ca82411
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2ca82411
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2ca82411
Branch: refs/heads/master
Commit: 2ca82411c33271261ce7d2df1d2de953b391e05b
Parents: dbc513e c04be63
Author: Li Yang <li...@apache.org>
Authored: Sun Oct 8 21:13:09 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Oct 8 21:13:09 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/cube/CubeSegment.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 2 +-
.../dict/MultipleDictionaryValueEnumerator.java | 50 ++++++++++--------
.../org/apache/kylin/dict/TrieDictionary.java | 2 +-
.../MultipleDictionaryValueEnumeratorTest.java | 54 ++++++++++++--------
.../kylin/metadata/model/TableExtDesc.java | 26 +++++++---
pom.xml | 2 +-
.../adhoc/JdbcPushDownConnectionManager.java | 2 +-
.../HiveColumnCardinalityUpdateJob.java | 6 ++-
.../apache/kylin/source/jdbc/JdbcExplorer.java | 1 +
.../source/kafka/config/KafkaClusterConfig.java | 5 +-
11 files changed, 95 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ca82411/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ca82411/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2ca82411/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --cc source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index be7b901,023ebfd..5ffa2f6
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@@ -122,7 -122,11 +122,11 @@@ public class HiveColumnCardinalityUpdat
tableExt.setCardinality(scardi);
metaMgr.saveTableExt(tableExt, prj);
} else {
- throw new IllegalArgumentException("No cardinality data is collected for table " + tableName);
+ // it gets here when ColumnCardinalityReducer output no record, which means empty table
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
++ TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj);
+ tableExt.resetCardinality();
+ metaMgr.saveTableExt(tableExt, prj);
}
}
[06/11] kylin git commit: KYLIN-2921 Refactor DataModelDesc,
split DataModelManager
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
index ccc8edc..d56cf6c 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java
@@ -36,9 +36,10 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
import org.apache.kylin.metadata.model.TableDesc;
@@ -182,8 +183,8 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
return CubeDescManager.getInstance(config);
}
- private static MetadataManager getMetadataManager(KylinConfig config) throws Exception {
- return MetadataManager.getInstance(config);
+ private static DataModelManager getMetadataManager(KylinConfig config) throws Exception {
+ return DataModelManager.getInstance(config);
}
@Test
@@ -300,42 +301,44 @@ public class CacheServiceTest extends LocalFileMetadataTestCase {
@Test
public void testMetaCRUD() throws Exception {
- final MetadataManager metadataManager = MetadataManager.getInstance(configA);
- final MetadataManager metadataManagerB = MetadataManager.getInstance(configB);
+ final TableMetadataManager tableMgr = TableMetadataManager.getInstance(configA);
+ final TableMetadataManager tableMgrB = TableMetadataManager.getInstance(configB);
+ final DataModelManager modelMgr = DataModelManager.getInstance(configA);
+ final DataModelManager modelMgrB = DataModelManager.getInstance(configB);
final Broadcaster broadcaster = Broadcaster.getInstance(configA);
broadcaster.getCounterAndClear();
TableDesc tableDesc = createTestTableDesc();
- assertTrue(metadataManager.getTableDesc(tableDesc.getIdentity(), "default") == null);
- assertTrue(metadataManagerB.getTableDesc(tableDesc.getIdentity(), "default") == null);
- metadataManager.saveSourceTable(tableDesc, "default");
+ assertTrue(tableMgr.getTableDesc(tableDesc.getIdentity(), "default") == null);
+ assertTrue(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default") == null);
+ tableMgr.saveSourceTable(tableDesc, "default");
//only one for table insert
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
- assertNotNull(metadataManager.getTableDesc(tableDesc.getIdentity(), "default"));
- assertNotNull(metadataManagerB.getTableDesc(tableDesc.getIdentity(), "default"));
+ assertNotNull(tableMgr.getTableDesc(tableDesc.getIdentity(), "default"));
+ assertNotNull(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default"));
final String dataModelName = "test_data_model";
- DataModelDesc dataModelDesc = metadataManager.getDataModelDesc("test_kylin_left_join_model_desc");
+ DataModelDesc dataModelDesc = modelMgr.getDataModelDesc("test_kylin_left_join_model_desc");
dataModelDesc.setName(dataModelName);
dataModelDesc.setLastModified(0);
- assertTrue(metadataManager.getDataModelDesc(dataModelName) == null);
- assertTrue(metadataManagerB.getDataModelDesc(dataModelName) == null);
+ assertTrue(modelMgr.getDataModelDesc(dataModelName) == null);
+ assertTrue(modelMgrB.getDataModelDesc(dataModelName) == null);
dataModelDesc.setName(dataModelName);
- metadataManager.createDataModelDesc(dataModelDesc, "default", "ADMIN");
+ modelMgr.createDataModelDesc(dataModelDesc, "default", "ADMIN");
//one for data model creation, one for project meta update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
- assertEquals(dataModelDesc.getName(), metadataManagerB.getDataModelDesc(dataModelName).getName());
+ assertEquals(dataModelDesc.getName(), modelMgrB.getDataModelDesc(dataModelName).getName());
final JoinTableDesc[] lookups = dataModelDesc.getJoinTables();
assertTrue(lookups.length > 0);
- metadataManager.updateDataModelDesc(dataModelDesc);
+ modelMgr.updateDataModelDesc(dataModelDesc);
//only one for data model update
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
- assertEquals(dataModelDesc.getJoinTables().length, metadataManagerB.getDataModelDesc(dataModelName).getJoinTables().length);
+ assertEquals(dataModelDesc.getJoinTables().length, modelMgrB.getDataModelDesc(dataModelName).getJoinTables().length);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
index 769d9d1..6cdd626 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/CubeServiceTest.java
@@ -50,7 +50,7 @@ public class CubeServiceTest extends ServiceTestBase {
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
Assert.assertNotNull(cubeService.getConfig());
Assert.assertNotNull(cubeService.getConfig());
- Assert.assertNotNull(cubeService.getMetadataManager());
+ Assert.assertNotNull(cubeService.getDataModelManager());
Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 44a30f1..65ad7fb 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -57,7 +57,7 @@ public class JobServiceTest extends ServiceTestBase {
public void testBasics() throws JobException, IOException, SQLException {
Assert.assertNotNull(jobService.getConfig());
Assert.assertNotNull(jobService.getConfig());
- Assert.assertNotNull(jobService.getMetadataManager());
+ Assert.assertNotNull(jobService.getDataModelManager());
Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
Assert.assertNull(jobService.getJobInstance("job_not_exist"));
Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL));
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index fb44b6c..760b6e3 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -23,12 +23,10 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.lang.reflect.Field;
import java.util.List;
+import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.junit.Assert;
@@ -49,13 +47,15 @@ public class ModelServiceTest extends ServiceTestBase {
@Test
public void testSuccessModelUpdate() throws IOException, JobException {
+ Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
+
List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
+ serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais));
+ DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));
deserialize.setOwner("somebody");
DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
@@ -63,66 +63,6 @@ public class ModelServiceTest extends ServiceTestBase {
}
@Test
- public void testSuccessModelUpdateOnComputedColumn()
- throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
-
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
- Assert.assertTrue(dataModelDescs.size() == 1);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais));
-
- Field field = ComputedColumnDesc.class.getDeclaredField("comment");
- field.setAccessible(true);
- field.set(deserialize.getComputedColumnDescs().get(0), "change on comment is okay");
- DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
- }
-
- @Test
- public void testFailureModelUpdateDueToComputedColumnConflict()
- throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
- expectedEx.expect(IllegalArgumentException.class);
- expectedEx.expectMessage(
- "Column name for computed column DEFAULT.TEST_KYLIN_FACT.DEAL_AMOUNT is already used in model ci_inner_join_model, you should apply the same expression ' PRICE * ITEM_COUNT ' here, or use a different column name.");
-
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
- Assert.assertTrue(dataModelDescs.size() == 1);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais));
-
- Field field = ComputedColumnDesc.class.getDeclaredField("expression");
- field.setAccessible(true);
- field.set(deserialize.getComputedColumnDescs().get(0), "another expression");
- DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
- }
-
- @Test
- public void testFailureModelUpdateDueToComputedColumnConflict2()
- throws IOException, JobException, NoSuchFieldException, IllegalAccessException {
- expectedEx.expect(IllegalArgumentException.class);
- expectedEx.expectMessage(
- "There is already a column named cal_dt on table DEFAULT.TEST_KYLIN_FACT, please change your computed column name");
-
- List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
- Assert.assertTrue(dataModelDescs.size() == 1);
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- MetadataManager.MODELDESC_SERIALIZER.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
- ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- DataModelDesc deserialize = MetadataManager.MODELDESC_SERIALIZER.deserialize(new DataInputStream(bais));
-
- Field field = ComputedColumnDesc.class.getDeclaredField("columnName");
- field.setAccessible(true);
- field.set(deserialize.getComputedColumnDescs().get(0), "cal_dt");
- DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
- }
-
- @Test
public void testRevisableModelInCaseOfDeleteMeasure() throws IOException {
List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_left_join_model", "default", true);
Assert.assertTrue(dataModelDescs.size() == 1);
@@ -136,7 +76,7 @@ public class ModelServiceTest extends ServiceTestBase {
expectedEx.expect(org.apache.kylin.rest.exception.BadRequestException.class);
expectedEx.expectMessage(
- "Measure: TEST_KYLIN_FACT.DEAL_AMOUNT can't be removed, It is referred in Cubes: [ci_left_join_cube]");
+ "Measure: TEST_KYLIN_FACT.ITEM_COUNT can't be removed, It is referred in Cubes: [ci_left_join_cube]");
modelService.updateModelToResourceStore(revisableModel, "default");
}
@@ -148,13 +88,14 @@ public class ModelServiceTest extends ServiceTestBase {
DataModelDesc revisableModel = dataModelDescs.get(0);
List<ModelDimensionDesc> originDims = revisableModel.getDimensions();
- String[] reviseDims = cutItems(originDims.get(0).getColumns(), 2);
+ String[] reviseDims = cutItems(originDims.get(0).getColumns(), 1);
originDims.get(0).setColumns(reviseDims);
revisableModel.setDimensions(originDims);
+ // actually the TEST_COUNT_DISTINCT_BITMAP was defined in dimension (not measure)
expectedEx.expect(org.apache.kylin.rest.exception.BadRequestException.class);
expectedEx.expectMessage(
- "Dimension: TEST_KYLIN_FACT.SELLER_ID_AND_COUNTRY_NAME can't be removed, It is referred in Cubes: [ci_left_join_cube]");
+ "Measure: TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP can't be removed, It is referred in Cubes: [ci_left_join_cube]");
modelService.updateModelToResourceStore(revisableModel, "default");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index c980959..884c3c8 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -48,7 +48,7 @@ public class QueryServiceTest extends ServiceTestBase {
public void testBasics() throws JobException, IOException, SQLException {
Assert.assertNotNull(queryService.getConfig());
Assert.assertNotNull(queryService.getConfig());
- Assert.assertNotNull(queryService.getMetadataManager());
+ Assert.assertNotNull(queryService.getDataModelManager());
Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
// Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 096134c..12916ec 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -48,7 +48,7 @@ import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -195,7 +195,7 @@ public class HiveMRInput implements IMRInput {
step.setName(ExecutableConstants.STEP_NAME_MATERIALIZE_HIVE_VIEW_IN_LOOKUP);
KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
- MetadataManager metadataManager = MetadataManager.getInstance(kylinConfig);
+ TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
final Set<TableDesc> lookupViewsTables = Sets.newHashSet();
String prj = flatDesc.getDataModel().getProject();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 229a6d7..55e6267 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -25,7 +25,7 @@ import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -49,7 +49,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String tableName, String prj) {
KylinConfig config = KylinConfig.getInstanceFromEnv();
- MetadataManager metaMgr = MetadataManager.getInstance(config);
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
HiveTableMeta hiveTableMeta;
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
index 1cadec4..da44ea5 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/ColumnCardinalityMapper.java
@@ -37,7 +37,7 @@ import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -63,7 +63,7 @@ public class ColumnCardinalityMapper<T> extends KylinMapper<T, Object, IntWritab
String project = conf.get(BatchConstants.CFG_PROJECT_NAME);
String tableName = conf.get(BatchConstants.CFG_TABLE_NAME);
- tableDesc = MetadataManager.getInstance(config).getTableDesc(tableName, project);
+ tableDesc = TableMetadataManager.getInstance(config).getTableDesc(tableName, project);
tableInputFormat = MRUtil.getTableInputFormat(tableDesc);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
index 5b3435f..3942a68 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityJob.java
@@ -35,7 +35,7 @@ import org.apache.kylin.engine.mr.MRUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +108,7 @@ public class HiveColumnCardinalityJob extends AbstractHadoopJob {
logger.info("Going to submit HiveColumnCardinalityJob for table '" + table + "'");
- TableDesc tableDesc = MetadataManager.getInstance(kylinConfig).getTableDesc(table, project);
+ TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(table, project);
attachTableMetadata(tableDesc, job.getConfiguration());
int result = waitForCompletion(job);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 7058ad8..be7b901 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +117,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
String scardi = cardi.toString();
if (scardi.length() > 0) {
scardi = scardi.substring(0, scardi.length() - 1);
- MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ TableMetadataManager metaMgr = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
TableExtDesc tableExt = metaMgr.getTableExt(tableName, prj);
tableExt.setCardinality(scardi);
metaMgr.saveTableExt(tableExt, prj);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
index fc6f878..46879b1 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java
@@ -22,9 +22,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.engine.mr.IMROutput;
import org.apache.kylin.engine.mr.IMROutput2;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.PartitionDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.IRealization;
@@ -70,7 +70,7 @@ public class HBaseStorage implements IStorage {
private static TblColRef getPartitionCol(IRealization realization) {
String modelName = realization.getModel().getName();
- DataModelDesc dataModelDesc = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
+ DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv()).getDataModelDesc(modelName);
PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
index ff8c487..7f55895 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ExtendCubeToHybridCLI.java
@@ -41,10 +41,10 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -74,7 +74,7 @@ public class ExtendCubeToHybridCLI {
private KylinConfig kylinConfig;
private CubeManager cubeManager;
private CubeDescManager cubeDescManager;
- private MetadataManager metadataManager;
+ private DataModelManager metadataManager;
private ResourceStore store;
public ExtendCubeToHybridCLI() {
@@ -82,7 +82,7 @@ public class ExtendCubeToHybridCLI {
this.store = ResourceStore.getStore(kylinConfig);
this.cubeManager = CubeManager.getInstance(kylinConfig);
this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
- this.metadataManager = MetadataManager.getInstance(kylinConfig);
+ this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
index 7d089ac..befe8a1 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/MeasureTypeOnlyAggrInBaseTest.java
@@ -35,9 +35,9 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.measure.MeasureAggregator;
import org.apache.kylin.measure.MeasureIngester;
import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.After;
import org.junit.Before;
@@ -65,7 +65,7 @@ public class MeasureTypeOnlyAggrInBaseTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
cube = getTestKylinCubeWithSeller();
cubeDesc = cube.getDescriptor();
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
index 9b1a00d..5bd5499 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/RowValueDecoderTest.java
@@ -30,9 +30,9 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.HBaseColumnDesc;
import org.apache.kylin.measure.BufferedMeasureCodec;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -42,7 +42,7 @@ public class RowValueDecoderTest extends LocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
this.createTestMetadata();
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
}
@After
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 865c0bc..9d8139d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -39,9 +39,9 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.exception.PersistentException;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -105,7 +105,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails");
private KylinConfig kylinConfig;
- private MetadataManager metadataManager;
+ private DataModelManager metadataManager;
private ProjectManager projectManager;
private HybridManager hybridManager;
private CubeManager cubeManager;
@@ -157,7 +157,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE) : null;
kylinConfig = KylinConfig.getInstanceFromEnv();
- metadataManager = MetadataManager.getInstance(kylinConfig);
+ metadataManager = DataModelManager.getInstance(kylinConfig);
projectManager = ProjectManager.getInstance(kylinConfig);
hybridManager = HybridManager.getInstance(kylinConfig);
cubeManager = CubeManager.getInstance(kylinConfig);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
index 999e703..40f68d0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
@@ -35,12 +35,13 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
@@ -72,11 +73,6 @@ public class CubeMetaIngester extends AbstractApplication {
private static final Option OPTION_OVERWRITE_TABLES = OptionBuilder.withArgName("overwriteTables").hasArg().isRequired(false).withDescription("If table meta conflicts, overwrite the one in metadata store with the one in srcPath. Use in caution because it might break existing cubes! Suggest to backup metadata store first").create("overwriteTables");
private KylinConfig kylinConfig;
- private MetadataManager metadataManager;
- private ProjectManager projectManager;
- private CubeManager cubeManager;
- private CubeDescManager cubeDescManager;
- private RealizationRegistry realizationRegistry;
Set<String> requiredResources = Sets.newLinkedHashSet();
private String targetProjectName;
@@ -96,11 +92,6 @@ public class CubeMetaIngester extends AbstractApplication {
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
kylinConfig = KylinConfig.getInstanceFromEnv();
- metadataManager = MetadataManager.getInstance(kylinConfig);
- projectManager = ProjectManager.getInstance(kylinConfig);
- cubeManager = CubeManager.getInstance(kylinConfig);
- cubeDescManager = CubeDescManager.getInstance(kylinConfig);
- realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
if (optionsHelper.hasOption(OPTION_FORCE_INGEST)) {
forceIngest = Boolean.valueOf(optionsHelper.getOptionValue(OPTION_FORCE_INGEST));
@@ -135,17 +126,19 @@ public class CubeMetaIngester extends AbstractApplication {
private void injest(File metaRoot) throws IOException {
KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
- MetadataManager srcMetadataManager = MetadataManager.getInstance(srcConfig);
+ TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
+ DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);
- checkAndMark(srcMetadataManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
+ checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
ResourceTool.copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));
- //clear the cache
- metadataManager.reload();
-
+ // clear the cache
+ Broadcaster.getInstance(kylinConfig).notifyClearAll();
+
+ ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
logger.info("add " + tableDesc + " to " + targetProjectName);
projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
@@ -159,16 +152,18 @@ public class CubeMetaIngester extends AbstractApplication {
}
- private void checkAndMark(MetadataManager srcMetadataManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
+ private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
if (srcHybridManager.listHybridInstances().size() > 0) {
throw new IllegalStateException("Does not support ingest hybrid yet");
}
+ ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
ProjectInstance targetProject = projectManager.getProject(targetProjectName);
if (targetProject == null) {
throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
}
+ TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
if (existing != null && !existing.equals(tableDesc)) {
@@ -185,8 +180,9 @@ public class CubeMetaIngester extends AbstractApplication {
requiredResources.add(tableDesc.getResourcePath());
}
- for (DataModelDesc dataModelDesc : srcMetadataManager.listDataModels()) {
- DataModelDesc existing = metadataManager.getDataModelDesc(dataModelDesc.getName());
+ DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
+ for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
+ DataModelDesc existing = modelManager.getDataModelDesc(dataModelDesc.getName());
if (existing != null) {
if (!forceIngest) {
throw new IllegalStateException("Already exist a model called " + dataModelDesc.getName());
@@ -197,6 +193,7 @@ public class CubeMetaIngester extends AbstractApplication {
requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
}
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
CubeDesc existing = cubeDescManager.getCubeDesc(cubeDesc.getName());
if (existing != null) {
@@ -209,6 +206,7 @@ public class CubeMetaIngester extends AbstractApplication {
requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
}
+ CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
for (CubeInstance cube : srcCubeManager.listAllCubes()) {
CubeInstance existing = cubeManager.getCube(cube.getName());
if (existing != null) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
index d048dd7..7517c07 100644
--- a/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/ExtendCubeToHybridCLI.java
@@ -41,10 +41,10 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.IStorageAware;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -72,7 +72,7 @@ public class ExtendCubeToHybridCLI {
private KylinConfig kylinConfig;
private CubeManager cubeManager;
private CubeDescManager cubeDescManager;
- private MetadataManager metadataManager;
+ private DataModelManager metadataManager;
private ResourceStore store;
public ExtendCubeToHybridCLI() {
@@ -80,7 +80,7 @@ public class ExtendCubeToHybridCLI {
this.store = ResourceStore.getStore(kylinConfig);
this.cubeManager = CubeManager.getInstance(kylinConfig);
this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
- this.metadataManager = MetadataManager.getInstance(kylinConfig);
+ this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
index 6555c4d..1471832 100644
--- a/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/KylinLogExtractor.java
@@ -29,7 +29,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.tool.util.ToolUtil;
import org.slf4j.Logger;
@@ -64,8 +64,8 @@ public class KylinLogExtractor extends AbstractInfoExtractor {
CubeManager.getInstance(config);
CubeDescManager.clearCache();
CubeDescManager.getInstance(config);
- MetadataManager.clearCache();
- MetadataManager.getInstance(config);
+ DataModelManager.clearCache();
+ DataModelManager.getInstance(config);
ProjectManager.clearCache();
ProjectManager.getInstance(config);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f45d8133/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
----------------------------------------------------------------------
diff --git a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
index ff12084..d974009 100644
--- a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
+++ b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
@@ -25,7 +25,7 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@ -66,7 +66,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase {
Assert.assertTrue(project.getModels().contains("cloned_model"));
Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube")));
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("cloned_cube");
@@ -83,7 +83,7 @@ public class CubeMetaIngesterTest extends LocalFileMetadataTestCase {
Assert.assertTrue(project.getModels().contains("benchmark_model"));
Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube")));
- MetadataManager.clearCache();
+ DataModelManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();
CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("benchmark_cube");