You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/14 09:00:06 UTC
[1/4] kylin git commit: minor, add cat in ResourceTool
Repository: kylin
Updated Branches:
refs/heads/2.x-staging e0cfca5f2 -> 81ca14e99
minor, add cat in ResourceTool
Signed-off-by: Li, Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f6212906
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f6212906
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f6212906
Branch: refs/heads/2.x-staging
Commit: f6212906177ebaff1c365b222af116038cd5ca00
Parents: e0cfca5
Author: Chen Luwei <ch...@hotmail.com>
Authored: Wed Jan 13 14:10:05 2016 +0800
Committer: Li, Yang <li...@apache.org>
Committed: Sun Feb 14 15:28:38 2016 +0800
----------------------------------------------------------------------
.../kylin/common/persistence/ResourceTool.java | 29 +++++++++
.../storage/hbase/steps/ResourceStoreTest.java | 63 ++++++++++++++++++++
2 files changed, 92 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6212906/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 8faeadf..d1661ac 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -19,6 +19,9 @@
package org.apache.kylin.common.persistence;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -68,11 +71,37 @@ public class ResourceTool {
case "remove":
remove(KylinConfig.getInstanceFromEnv(), args[1]);
break;
+ case "cat":
+ cat(KylinConfig.getInstanceFromEnv(), args[1]);
+ break;
default:
System.out.println("Unknown cmd: " + cmd);
}
}
+ public static void cat(KylinConfig config, String path) throws IOException {
+ ResourceStore store = ResourceStore.getStore(config);
+ InputStream is = store.getResource(path).inputStream;
+ BufferedReader br = null;
+ String line;
+ try {
+ br = new BufferedReader(new InputStreamReader(is));
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (br != null) {
+ try {
+ br.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
public static void list(KylinConfig config, String path) throws IOException {
ResourceStore store = ResourceStore.getStore(config);
ArrayList<String> result = store.listResources(path);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f6212906/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
new file mode 100644
index 0000000..020394b
--- /dev/null
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.hbase.steps;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.ClassUtil;
+
+import java.io.File;
+
+/**
+ * This is a helper class for developer to directly manipulate the metadata store in sandbox
+ * This is designed to run in IDE(i.e. not on sandbox hadoop CLI)
+ *
+ * For production metadata store manipulation refer to bin/metastore.sh in binary package
+ * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
+ */
+public class ResourceStoreTest {
+
+ private static final Log logger = LogFactory.getLog(ResourceStoreTest.class);
+
+ public static void main(String[] args) throws Exception {
+ logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
+ System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
+ if (System.getProperty("hdp.version") == null) {
+ throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
+ }
+
+ if (args.length < 2) {
+ printUsage();
+ return;
+ }
+
+ if ("cat".equalsIgnoreCase(args[0])) {
+ ResourceTool.main(new String[] { "cat", args[1] });
+ } else {
+ printUsage();
+ }
+ }
+
+ private static void printUsage() {
+ logger.info("Usage: ResourceStoreTest cat file");
+ }
+}
[4/4] kylin git commit: KYLIN-1405 code review edits and other minor
refactor
Posted by li...@apache.org.
KYLIN-1405 code review edits and other minor refactor
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/81ca14e9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/81ca14e9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/81ca14e9
Branch: refs/heads/2.x-staging
Commit: 81ca14e991e9a8d86970bb2a643aedd50859f6ff
Parents: e79e6b4
Author: Li, Yang <li...@apache.org>
Authored: Sun Feb 14 15:59:46 2016 +0800
Committer: Li, Yang <li...@apache.org>
Committed: Sun Feb 14 15:59:46 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 12 ++++-
.../kylin/common/persistence/ResourceTool.java | 16 ++-----
.../org/apache/kylin/cube/model/CubeDesc.java | 48 ++++++++++----------
.../validation/rule/AggregationGroupRule.java | 3 +-
.../org/apache/kylin/cube/CubeDescTest.java | 15 +++---
.../kylin/rest/security/CrossDomainFilter.java | 2 +-
6 files changed, 46 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 5ce4ddc..bd70249 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -73,11 +73,11 @@ public class KylinConfigBase implements Serializable {
private volatile Properties properties = new Properties();
- public String getOptional(String prop) {
+ protected String getOptional(String prop) {
return getOptional(prop, null);
}
- public String getOptional(String prop, String dft) {
+ protected String getOptional(String prop, String dft) {
final String property = System.getProperty(prop);
return property != null ? property : properties.getProperty(prop, dft);
}
@@ -375,6 +375,10 @@ public class KylinConfigBase implements Serializable {
public double getCubeAlgorithmAutoThreshold() {
return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
}
+
+ public int getCubeAggrGroupMaxSize() {
+ return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max_size", "12"));
+ }
public int getDictionaryMaxCardinality() {
return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
@@ -529,6 +533,10 @@ public class KylinConfigBase implements Serializable {
public String getMailSender() {
return getOptional("mail.sender", "");
}
+
+ public boolean isWebCrossDomainEnabled() {
+ return Boolean.parseBoolean(getOptional("crossdomain.enable", "true"));
+ }
public String toString() {
return getMetadataUrl();
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index d1661ac..3b8e0c1 100644
--- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -18,15 +18,14 @@
package org.apache.kylin.common.persistence;
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
-import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Callable;
-import com.google.common.base.Function;
+import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.StringUtil;
@@ -89,16 +88,9 @@ public class ResourceTool {
while ((line = br.readLine()) != null) {
System.out.println(line);
}
- } catch (IOException e) {
- e.printStackTrace();
} finally {
- if (br != null) {
- try {
- br.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
+ IOUtils.closeQuietly(is);
+ IOUtils.closeQuietly(br);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index be5e4ee..5a345b9 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -20,8 +20,18 @@ package org.apache.kylin.cube.model;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
import javax.annotation.Nullable;
@@ -35,9 +45,6 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.model.validation.IValidatorRule;
-import org.apache.kylin.cube.model.validation.ResultLevel;
-import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
@@ -51,6 +58,8 @@ import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
@@ -60,8 +69,6 @@ import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*/
@@ -478,7 +485,7 @@ public class CubeDesc extends RootPersistentEntity {
}
// check if aggregation group is valid
- validate(this);
+ validate();
this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
@@ -511,15 +518,11 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- public void validate(CubeDesc cube) {
- inner(cube);
- }
-
- private void inner(CubeDesc cube) {
- int maxSize = getMaxAgrGroupSize();
+ public void validate() {
+ int maxSize = config.getCubeAggrGroupMaxSize();
int index = 0;
- for (AggregationGroup agg : cube.getAggregationGroups()) {
+ for (AggregationGroup agg : getAggregationGroups()) {
if (agg.getIncludes() == null) {
logger.error("Aggregation group " + index + " includes field not set");
throw new IllegalStateException("Aggregation group " + index + " includes field not set");
@@ -536,11 +539,11 @@ public class CubeDesc extends RootPersistentEntity {
Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
getDims(mandatoryDims, agg.getSelectRule().mandatory_dims);
- ArrayList<Set<String>> hierarchyDimsList = new ArrayList ();
+ ArrayList<Set<String>> hierarchyDimsList = Lists.newArrayList();
Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
getDims(hierarchyDimsList, hierarchyDims, agg.getSelectRule().hierarchy_dims);
- ArrayList<Set<String>> jointDimsList = new ArrayList ();
+ ArrayList<Set<String>> jointDimsList = Lists.newArrayList();
Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
@@ -595,7 +598,7 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- private void getDims (Set<String> dims, String [] stringSet) {
+ private void getDims(Set<String> dims, String[] stringSet) {
if (stringSet != null) {
for (String str : stringSet) {
dims.add(str);
@@ -603,7 +606,7 @@ public class CubeDesc extends RootPersistentEntity {
}
}
- private void getDims (ArrayList<Set<String>> dimsList, Set<String> dims, String [][] stringSets) {
+ private void getDims(ArrayList<Set<String>> dimsList, Set<String> dims, String[][] stringSets) {
Set<String> temp = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
if (stringSets != null) {
for (String[] ss : stringSets) {
@@ -626,7 +629,7 @@ public class CubeDesc extends RootPersistentEntity {
return b;
}
- private boolean hasSingle (ArrayList<Set<String>> dimsList) {
+ private boolean hasSingle(ArrayList<Set<String>> dimsList) {
boolean hasSingle = false;
for (Set<String> dims : dimsList) {
if (dims.size() < 2)
@@ -635,7 +638,7 @@ public class CubeDesc extends RootPersistentEntity {
return hasSingle;
}
- private boolean hasOverlap (ArrayList<Set<String>> dimsList, Set<String> Dims) {
+ private boolean hasOverlap(ArrayList<Set<String>> dimsList, Set<String> Dims) {
boolean hasOverlap = false;
int dimSize = 0;
for (Set<String> dims : dimsList) {
@@ -646,11 +649,6 @@ public class CubeDesc extends RootPersistentEntity {
return hasOverlap;
}
- protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getOptional(IValidatorRule.KEY_MAX_AGR_GROUP_SIZE, String.valueOf(IValidatorRule.DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
- }
-
private void initDimensionColumns() {
for (DimensionDesc dim : dimensions) {
JoinDesc join = dim.getJoin();
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
index 222288a..196d2e8 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/validation/rule/AggregationGroupRule.java
@@ -179,7 +179,6 @@ public class AggregationGroupRule implements IValidatorRule<CubeDesc> {
}
protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getOptional(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
+ return KylinConfig.getInstanceFromEnv().getCubeAggrGroupMaxSize();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 512f99f..7013cff 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -97,14 +97,13 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
thrown.expect(IllegalStateException.class);
thrown.expectMessage("Aggregation group 0 has too many dimensions");
- CubeDesc desc = new CubeDesc() {
- @Override
- protected int getMaxAgrGroupSize() {
- return 3;
- }
- };
- CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
- desc.validate(cubeDesc);
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ try {
+ System.setProperty("kylin.cube.aggrgroup.max_size", "3");
+ cubeDesc.validate();
+ } finally {
+ System.clearProperty("kylin.cube.aggrgroup.max_size");
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/81ca14e9/server/src/main/java/org/apache/kylin/rest/security/CrossDomainFilter.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/CrossDomainFilter.java b/server/src/main/java/org/apache/kylin/rest/security/CrossDomainFilter.java
index c9167b9..7d9d9ac 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/CrossDomainFilter.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/CrossDomainFilter.java
@@ -53,7 +53,7 @@ public class CrossDomainFilter implements Filter {
*/
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
- if (Boolean.parseBoolean(KylinConfig.getInstanceFromEnv().getOptional("crossdomain.enable", "true"))) {
+ if (KylinConfig.getInstanceFromEnv().isWebCrossDomainEnabled()) {
((HttpServletResponse) response).addHeader("Access-Control-Allow-Origin", "*");
((HttpServletResponse) response).addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
((HttpServletResponse) response).addHeader("Access-Control-Allow-Headers", "Origin, No-Cache, X-Requested-With, If-Modified-Since, Pragma, Last-Modified, Cache-Control, Expires, Content-Type, X-E4M-With, Accept, Authorization");
[2/4] kylin git commit: KYLIN-1405 add validation when init cube desc
Posted by li...@apache.org.
KYLIN-1405 add validation when init cube desc
Signed-off-by: Li, Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/64800a10
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/64800a10
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/64800a10
Branch: refs/heads/2.x-staging
Commit: 64800a107b7c24bffe8e8803e85c0bdfd2b407b4
Parents: f621290
Author: Chen Luwei <ch...@hotmail.com>
Authored: Tue Jan 26 11:03:44 2016 +0800
Committer: Li, Yang <li...@apache.org>
Committed: Sun Feb 14 15:28:44 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 161 +++++++++++++++--
.../org/apache/kylin/cube/CubeDescTest.java | 172 ++++++++++++++++++-
2 files changed, 315 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/64800a10/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index a1a0531..be5e4ee 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -20,21 +20,13 @@ package org.apache.kylin.cube.model;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Set;
import javax.annotation.Nullable;
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
@@ -43,6 +35,9 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Array;
import org.apache.kylin.common.util.CaseInsensitiveStringMap;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.validation.IValidatorRule;
+import org.apache.kylin.cube.model.validation.ResultLevel;
+import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.metadata.MetadataConstants;
@@ -65,12 +60,15 @@ import com.google.common.base.Function;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*/
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
public class CubeDesc extends RootPersistentEntity {
+ private static final Logger logger = LoggerFactory.getLogger(CubeDesc.class);
public static class CannotFilterExtendedColumnException extends RuntimeException {
public CannotFilterExtendedColumnException(TblColRef tblColRef) {
@@ -479,6 +477,9 @@ public class CubeDesc extends RootPersistentEntity {
this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
}
+ // check if aggregation group is valid
+ validate(this);
+
this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
if (this.model == null) {
@@ -510,6 +511,146 @@ public class CubeDesc extends RootPersistentEntity {
}
}
+ public void validate(CubeDesc cube) {
+ inner(cube);
+ }
+
+ private void inner(CubeDesc cube) {
+ int maxSize = getMaxAgrGroupSize();
+ int index = 0;
+
+ for (AggregationGroup agg : cube.getAggregationGroups()) {
+ if (agg.getIncludes() == null) {
+ logger.error("Aggregation group " + index + " includes field not set");
+ throw new IllegalStateException("Aggregation group " + index + " includes field not set");
+ }
+
+ if (agg.getSelectRule() == null) {
+ logger.error("Aggregation group " + index + " includes field not set");
+ throw new IllegalStateException("Aggregation group " + index + " select rule field not set");
+ }
+
+ Set<String> includeDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ getDims(includeDims, agg.getIncludes());
+
+ Set<String> mandatoryDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ getDims(mandatoryDims, agg.getSelectRule().mandatory_dims);
+
+ ArrayList<Set<String>> hierarchyDimsList = new ArrayList ();
+ Set<String> hierarchyDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ getDims(hierarchyDimsList, hierarchyDims, agg.getSelectRule().hierarchy_dims);
+
+ ArrayList<Set<String>> jointDimsList = new ArrayList ();
+ Set<String> jointDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ getDims(jointDimsList, jointDims, agg.getSelectRule().joint_dims);
+
+ if (!includeDims.containsAll(mandatoryDims) || !containsAll(includeDims, hierarchyDimsList) || !containsAll(includeDims, jointDimsList)) {
+ logger.error("Aggregation group " + index + " Include dims not containing all the used dims");
+ throw new IllegalStateException("Aggregation group " + index + " Include dims not containing all the used dims");
+ }
+
+ Set<String> normalDims = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ normalDims.addAll(includeDims);
+ normalDims.removeAll(mandatoryDims);
+ normalDims.removeAll(hierarchyDims);
+ normalDims.removeAll(jointDims);
+
+ int normalDimSize = normalDims.size();
+ int hierarchyDimSize = hierarchyDimsList.size();
+ int jointDimSize = jointDimsList.size();
+
+ if (normalDimSize + hierarchyDimSize + jointDimSize > maxSize) {
+ logger.error("Aggregation group " + index + " has too many dimensions");
+ throw new IllegalStateException("Aggregation group " + index + " has too many dimensions");
+ }
+
+ if (CollectionUtils.containsAny(mandatoryDims, hierarchyDims)) {
+ logger.warn("Aggregation group " + index + " mandatory dims overlap with hierarchy dims");
+ }
+ if (CollectionUtils.containsAny(mandatoryDims, jointDims)) {
+ logger.warn("Aggregation group " + index + " mandatory dims overlap with joint dims");
+ }
+ if (CollectionUtils.containsAny(hierarchyDims, jointDims)) {
+ logger.error("Aggregation group " + index + " hierarchy dims overlap with joint dims");
+ throw new IllegalStateException("Aggregation group " + index + " hierarchy dims overlap with joint dims");
+ }
+ if (hasSingle(hierarchyDimsList)) {
+ logger.error("Aggregation group " + index + " require at least 2 dims in a hierarchy");
+ throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a hierarchy");
+ }
+ if (hasSingle(jointDimsList)) {
+ logger.error("Aggregation group " + index + " require at least 2 dims in a joint");
+ throw new IllegalStateException("Aggregation group " + index + " require at least 2 dims in a joint");
+ }
+ if (hasOverlap(hierarchyDimsList, hierarchyDims)) {
+ logger.error("Aggregation group " + index + " a dim exist in more than one hierarchy");
+ throw new IllegalStateException("Aggregation group " + index + " a dim exist in more than one hierarchy");
+ }
+ if (hasOverlap(jointDimsList, jointDims)) {
+ logger.error("Aggregation group " + index + " a dim exist in more than one joint");
+ throw new IllegalStateException("Aggregation group " + index + " a dim exist in more than one joint");
+ }
+
+ index++;
+ }
+ }
+
+ private void getDims (Set<String> dims, String [] stringSet) {
+ if (stringSet != null) {
+ for (String str : stringSet) {
+ dims.add(str);
+ }
+ }
+ }
+
+ private void getDims (ArrayList<Set<String>> dimsList, Set<String> dims, String [][] stringSets) {
+ Set<String> temp = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (stringSets != null) {
+ for (String[] ss : stringSets) {
+ temp.clear();
+ for (String s : ss) {
+ temp.add(s);
+ dims.add(s);
+ }
+ dimsList.add(temp);
+ }
+ }
+ }
+
+ private boolean containsAll(Set<String> includeDims, ArrayList<Set<String>> dimsList) {
+ boolean b = true;
+ for (Set<String> dims : dimsList) {
+ if (!includeDims.containsAll(dims))
+ b = false;
+ }
+ return b;
+ }
+
+ private boolean hasSingle (ArrayList<Set<String>> dimsList) {
+ boolean hasSingle = false;
+ for (Set<String> dims : dimsList) {
+ if (dims.size() < 2)
+ hasSingle = true;
+ }
+ return hasSingle;
+ }
+
+ private boolean hasOverlap (ArrayList<Set<String>> dimsList, Set<String> Dims) {
+ boolean hasOverlap = false;
+ int dimSize = 0;
+ for (Set<String> dims : dimsList) {
+ dimSize += dims.size();
+ }
+ if (dimSize != Dims.size())
+ hasOverlap = true;
+ return hasOverlap;
+ }
+
+ protected int getMaxAgrGroupSize() {
+ String size = KylinConfig.getInstanceFromEnv().getOptional(IValidatorRule.KEY_MAX_AGR_GROUP_SIZE, String.valueOf(IValidatorRule.DEFAULT_MAX_AGR_GROUP_SIZE));
+ return Integer.parseInt(size);
+ }
+
private void initDimensionColumns() {
for (DimensionDesc dim : dimensions) {
JoinDesc join = dim.getJoin();
http://git-wip-us.apache.org/repos/asf/kylin/blob/64800a10/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index 13fcd4a..512f99f 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -18,24 +18,28 @@
package org.apache.kylin.cube;
-import java.util.HashMap;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.metadata.MetadataManager;
+import org.junit.*;
+import org.junit.rules.ExpectedException;
-import com.google.common.collect.Maps;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
/**
* @author yangli9
*/
public class CubeDescTest extends LocalFileMetadataTestCase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+
@Before
public void setUp() throws Exception {
this.createTestMetadata();
@@ -47,6 +51,158 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
}
@Test
+ public void testGoodInit() throws Exception {
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit1() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 includes field not set");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ String[] temp = null;
+ cubeDesc.getAggregationGroups().get(0).setIncludes(temp);
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit2() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 select rule field not set");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ SelectRule temp = null;
+ cubeDesc.getAggregationGroups().get(0).setSelectRule(temp);
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit3() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 Include dims not containing all the used dims");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]);
+ cubeDesc.getAggregationGroups().get(0).setIncludes(temp);
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit4() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 has too many dimensions");
+
+ CubeDesc desc = new CubeDesc() {
+ @Override
+ protected int getMaxAgrGroupSize() {
+ return 3;
+ }
+ };
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ desc.validate(cubeDesc);
+ }
+
+ @Test
+ public void testBadInit5() throws Exception {
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatory_dims = new String[] {
+ "seller_id", "META_CATEG_NAME"};
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit6() throws Exception {
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatory_dims = new String[] {
+ "seller_id", "lstg_format_name"};
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit7() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 require at least 2 dims in a joint");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {
+ new String[] { "lstg_format_name" } };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit8() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 hierarchy dims overlap with joint dims");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {
+ new String[] { "META_CATEG_NAME", "CATEG_LVL2_NAME" } };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit9() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 hierarchy dims overlap with joint dims");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchy_dims = new String[][] {
+ new String[] { "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" },
+ new String[] { "lstg_format_name", "lstg_site_id" } };
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {
+ new String[] { "META_CATEG_NAME", "lstg_format_name" } };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit10() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 a dim exist in more than one joint");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().joint_dims = new String[][] {
+ new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd" },
+ new String[] { "lstg_format_name", "lstg_site_id", "leaf_categ_id"} };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit11() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 require at least 2 dims in a hierarchy");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchy_dims = new String[][] {
+ new String[] { "META_CATEG_NAME" } };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
+ public void testBadInit12() throws Exception {
+ thrown.expect(IllegalStateException.class);
+ thrown.expectMessage("Aggregation group 0 a dim exist in more than one hierarchy");
+
+ CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
+ cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchy_dims = new String[][] {
+ new String[] { "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME" },
+ new String[] { "META_CATEG_NAME", "CATEG_LVL2_NAME" } };
+
+ cubeDesc.init(getTestConfig(), MetadataManager.getInstance(getTestConfig()).getAllTablesMap());
+ }
+
+ @Test
public void testSerialize() throws Exception {
CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
String str = JsonUtil.writeValueAsIndentString(desc);
[3/4] kylin git commit: minor,
delete ResourceStoreTest that is only useful to launch cat in
ResourceTool
Posted by li...@apache.org.
minor, delete ResourceStoreTest that is only useful to launch cat in ResourceTool
Signed-off-by: Li, Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e79e6b4d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e79e6b4d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e79e6b4d
Branch: refs/heads/2.x-staging
Commit: e79e6b4dc367d945c52ceddfd9a119e848564a24
Parents: 64800a1
Author: Chen Luwei <ch...@hotmail.com>
Authored: Thu Jan 28 18:27:15 2016 +0800
Committer: Li, Yang <li...@apache.org>
Committed: Sun Feb 14 15:28:48 2016 +0800
----------------------------------------------------------------------
.../storage/hbase/steps/ResourceStoreTest.java | 63 --------------------
1 file changed, 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e79e6b4d/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
deleted file mode 100644
index 020394b..0000000
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/ResourceStoreTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.hbase.steps;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceTool;
-import org.apache.kylin.common.util.ClassUtil;
-
-import java.io.File;
-
-/**
- * This is a helper class for developer to directly manipulate the metadata store in sandbox
- * This is designed to run in IDE(i.e. not on sandbox hadoop CLI)
- *
- * For production metadata store manipulation refer to bin/metastore.sh in binary package
- * It is desinged to run in hadoop CLI, both in sandbox or in real hadoop environment
- */
-public class ResourceStoreTest {
-
- private static final Log logger = LogFactory.getLog(ResourceStoreTest.class);
-
- public static void main(String[] args) throws Exception {
- logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
- System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
- if (System.getProperty("hdp.version") == null) {
- throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
- }
-
- if (args.length < 2) {
- printUsage();
- return;
- }
-
- if ("cat".equalsIgnoreCase(args[0])) {
- ResourceTool.main(new String[] { "cat", args[1] });
- } else {
- printUsage();
- }
- }
-
- private static void printUsage() {
- logger.info("Usage: ResourceStoreTest cat file");
- }
-}