You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by lu...@apache.org on 2015/01/14 15:15:46 UTC
[11/51] [partial] incubator-kylin git commit: cleanup for migration
from github.com
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/cube/TblColRef.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/cube/TblColRef.java b/metadata/src/main/java/com/kylinolap/metadata/model/cube/TblColRef.java
deleted file mode 100644
index a57c209..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/cube/TblColRef.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata.model.cube;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.kylinolap.metadata.model.schema.ColumnDesc;
-import com.kylinolap.metadata.model.schema.DataType;
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/26/13 Time: 1:30 PM To
- * change this template use File | Settings | File Templates.
- */
-public class TblColRef {
-
- private static final String INNER_TABLE_NAME = "_kylin_table";
-
- // used by projection rewrite, see OLAPProjectRel
- public enum InnerDataTypeEnum {
-
- LITERAL("_literal_type"), DERIVED("_derived_type");
-
- private final String dateType;
-
- private InnerDataTypeEnum(String name) {
- this.dateType = name;
- }
-
- public String getDataType() {
- return dateType;
- }
-
- public static boolean contains(String name) {
- return LITERAL.getDataType().equals(name) || DERIVED.getDataType().equals(name);
- }
- }
-
- // used by projection rewrite, see OLAPProjectRel
- public static TblColRef newInnerColumn(String columnName, InnerDataTypeEnum dataType) {
- ColumnDesc column = new ColumnDesc();
- column.setName(columnName);
- TableDesc table = new TableDesc();
- column.setTable(table);
- TblColRef colRef = new TblColRef(column);
- colRef.markInnerColumn(dataType);
- return colRef;
- }
-
- // ============================================================================
-
- private ColumnDesc column;
-
- public TblColRef(ColumnDesc column) {
- this.column = column;
- }
-
- public ColumnDesc getColumn() {
- return column;
- }
-
- public void setColumn(ColumnDesc column) {
- this.column = column;
- }
-
- public String getName() {
- return column.getName();
- }
-
- public String getTable() {
- if (column.getTable() == null) {
- return null;
- }
- return column.getTable().getName();
- }
-
- public String getDatatype() {
- return column.getDatatype();
- }
-
- public DataType getType() {
- return column.getType();
- }
-
- public void markInnerColumn(InnerDataTypeEnum dataType) {
- this.column.setDatatype(dataType.getDataType());
- this.column.getTable().setName(INNER_TABLE_NAME);
- }
-
- public boolean isInnerColumn() {
- return InnerDataTypeEnum.contains(getDatatype());
- }
-
- public boolean isDerivedDataType() {
- return InnerDataTypeEnum.DERIVED.getDataType().equals(getDatatype());
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
-
- result = prime * result + column.getTable().getName().hashCode();
- result = prime * result + column.getName().hashCode();
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TblColRef other = (TblColRef) obj;
- if (!StringUtils.equals(column.getTable().getName(), other.column.getTable().getName()))
- return false;
- if (!StringUtils.equals(column.getName(), other.column.getName()))
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return (column.getTable() == null ? null : column.getTable().getName()) + "." + column.getName();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/invertedindex/InvertedIndexDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/invertedindex/InvertedIndexDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/invertedindex/InvertedIndexDesc.java
deleted file mode 100644
index 324adb7..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/invertedindex/InvertedIndexDesc.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package com.kylinolap.metadata.model.invertedindex;
-
-import java.util.BitSet;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.persistence.ResourceStore;
-import com.kylinolap.common.persistence.RootPersistentEntity;
-import com.kylinolap.common.util.StringUtil;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.TblColRef;
-import com.kylinolap.metadata.model.schema.ColumnDesc;
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * @author yangli9
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class InvertedIndexDesc extends RootPersistentEntity {
-
- public static final String HBASE_FAMILY = "f";
- public static final String HBASE_QUALIFIER = "c";
- public static final byte[] HBASE_FAMILY_BYTES = Bytes.toBytes(HBASE_FAMILY);
- public static final byte[] HBASE_QUALIFIER_BYTES = Bytes.toBytes(HBASE_QUALIFIER);
-
- private KylinConfig config;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("fact_table")
- private String factTable;
- @JsonProperty("timestamp_dimension")
- private String timestampDimension;
- @JsonProperty("bitmap_dimensions")
- private String[] bitmapDimensions;
- @JsonProperty("value_dimensions")
- private String[] valueDimensions;
- @JsonProperty("metrics")
- private String[] metrics;
- @JsonProperty("sharding")
- private short sharding = 1; // parallelism
- @JsonProperty("slice_size")
- private int sliceSize = 50000; // no. rows
-
- // computed
- private TableDesc tableDesc;
- private int tsCol;
- private int[] bitmapCols;
- private int[] valueCols;
- private int[] metricsCols;
- private BitSet metricsColSet;
-
- public void init(MetadataManager mgr) {
- config = mgr.getConfig();
-
- factTable = factTable.toUpperCase();
- timestampDimension = timestampDimension.toUpperCase();
- StringUtil.toUpperCaseArray(bitmapDimensions, bitmapDimensions);
- StringUtil.toUpperCaseArray(valueDimensions, valueDimensions);
- StringUtil.toUpperCaseArray(metrics, metrics);
-
- tableDesc = mgr.getTableDesc(factTable);
- bitmapCols = new int[bitmapDimensions.length];
- valueCols = new int[valueDimensions.length];
- metricsCols = new int[metrics.length];
- metricsColSet = new BitSet(tableDesc.getColumnCount());
- int i = 0, j = 0, k = 0;
- for (ColumnDesc col : tableDesc.getColumns()) {
- if (ArrayUtils.contains(bitmapDimensions, col.getName())) {
- bitmapCols[i++] = col.getZeroBasedIndex();
- }
- if (ArrayUtils.contains(valueDimensions, col.getName())) {
- valueCols[j++] = col.getZeroBasedIndex();
- }
- if (ArrayUtils.contains(metrics, col.getName())) {
- metricsCols[k++] = col.getZeroBasedIndex();
- metricsColSet.set(col.getZeroBasedIndex());
- }
- }
-
- tsCol = tableDesc.findColumnByName(timestampDimension).getZeroBasedIndex();
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public String getName() {
- return name;
- }
-
- public int getTimestampColumn() {
- return tsCol;
- }
-
- public int[] getBitmapColumns() {
- return bitmapCols;
- }
-
- public int[] getValueColumns() {
- return valueCols;
- }
-
- public int[] getMetricsColumns() {
- return metricsCols;
- }
-
- public short getSharding() {
- return sharding;
- }
-
- public int getSliceSize() {
- return sliceSize;
- }
-
- public boolean isMetricsCol(TblColRef col) {
- assert col.getTable().equals(factTable);
- return isMetricsCol(col.getColumn().getZeroBasedIndex());
- }
-
- public boolean isMetricsCol(int colZeroBasedIndex) {
- return metricsColSet.get(colZeroBasedIndex);
- }
-
- public String getResourcePath() {
- return ResourceStore.IIDESC_RESOURCE_ROOT + "/" + name + ".json";
- }
-
- public TableDesc getFactTableDesc() {
- return tableDesc;
- }
-
- public String getFactTable() {
- return factTable;
- }
-
- public String getTimestampDimension() {
- return timestampDimension;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/schema/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/schema/ColumnDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/schema/ColumnDesc.java
deleted file mode 100644
index 1facf04..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/schema/ColumnDesc.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata.model.schema;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Column Metadata from Source. All name should be uppercase.
- * <p/>
- * User: lukhan Date: 10/15/13 Time: 9:07 AM To change this template use File |
- * Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class ColumnDesc {
- @JsonProperty("id")
- private String id;
- @JsonProperty("name")
- private String name;
- @JsonProperty("datatype")
- private String datatype;
-
- // parsed from data type
- private DataType type;
-
- private TableDesc table;
- private int zeroBasedIndex = -1;
- private boolean isNullable = true;
-
- public ColumnDesc() { // default constructor for Jackson
- }
-
- public int getZeroBasedIndex() {
- return zeroBasedIndex;
- }
-
- public String getDatatype() {
- return datatype;
- }
-
- public void setDatatype(String datatype) {
- this.datatype = datatype;
- type = DataType.getInstance(datatype);
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public TableDesc getTable() {
- return table;
- }
-
- public void setTable(TableDesc table) {
- this.table = table;
- }
-
- public DataType getType() {
- return type;
- }
-
- public String getTypeName() {
- return type.getName();
- }
-
- public int getTypePrecision() {
- return type.getPrecision();
- }
-
- public int getTypeScale() {
- return type.getScale();
- }
-
- public void setNullable(boolean nullable) {
- this.isNullable = nullable;
- }
-
- public boolean isNullable() {
- return this.isNullable;
- }
-
- public void init(TableDesc table) {
- this.table = table;
-
- if (name != null)
- name = name.toUpperCase();
-
- if (id != null)
- zeroBasedIndex = Integer.parseInt(id) - 1;
-
- type = DataType.getInstance(datatype);
- }
-
- @Override
- public String toString() {
- return "ColumnDesc [name=" + name + ",table=" + table.getName() + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/schema/DataType.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/schema/DataType.java b/metadata/src/main/java/com/kylinolap/metadata/model/schema/DataType.java
deleted file mode 100644
index b955fae..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/schema/DataType.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.model.schema;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import com.kylinolap.metadata.model.cube.TblColRef.InnerDataTypeEnum;
-
-/**
- * @author yangli9
- *
- */
-public class DataType {
-
- public static final String VALID_TYPES_STRING = "any|char|varchar|boolean|binary" //
- + "|integer|tinyint|smallint|bigint|decimal|numeric|float|real|double" //
- + "|date|time|datetime|timestamp|byte|int|short|long|string|hllc" //
- + "|" + InnerDataTypeEnum.LITERAL.getDataType() //
- + "|" + InnerDataTypeEnum.DERIVED.getDataType();
-
- private static final Pattern TYPE_PATTERN = Pattern.compile(
- // standard sql types, ref:
- // http://www.w3schools.com/sql/sql_datatypes_general.asp
- "(" + VALID_TYPES_STRING + ")" + "\\s*" //
- + "(?:" + "[(]" + "([\\d\\s,]+)" + "[)]" + ")?", Pattern.CASE_INSENSITIVE);
-
- public static final Set<String> INTEGER_FAMILY = new HashSet<String>();
- public static final Set<String> NUMBER_FAMILY = new HashSet<String>();
- public static final Set<String> DATETIME_FAMILY = new HashSet<String>();
- public static final Set<String> STRING_FAMILY = new HashSet<String>();
- private static final Set<Integer> HLLC_PRECISIONS = new HashSet<Integer>();
- private static final Map<String, String> LEGACY_TYPE_MAP = new HashMap<String, String>();
- static {
- INTEGER_FAMILY.add("tinyint");
- INTEGER_FAMILY.add("smallint");
- INTEGER_FAMILY.add("integer");
- INTEGER_FAMILY.add("bigint");
-
- NUMBER_FAMILY.addAll(INTEGER_FAMILY);
- NUMBER_FAMILY.add("float");
- NUMBER_FAMILY.add("double");
- NUMBER_FAMILY.add("decimal");
- NUMBER_FAMILY.add("real");
- NUMBER_FAMILY.add("numeric");
-
- DATETIME_FAMILY.add("date");
- DATETIME_FAMILY.add("time");
- DATETIME_FAMILY.add("datetime");
- DATETIME_FAMILY.add("timestamp");
-
- STRING_FAMILY.add("varchar");
- STRING_FAMILY.add("char");
-
- LEGACY_TYPE_MAP.put("byte", "tinyint");
- LEGACY_TYPE_MAP.put("int", "integer");
- LEGACY_TYPE_MAP.put("short", "smallint");
- LEGACY_TYPE_MAP.put("long", "bigint");
- LEGACY_TYPE_MAP.put("string", "varchar");
- LEGACY_TYPE_MAP.put("hllc10", "hllc(10)");
- LEGACY_TYPE_MAP.put("hllc12", "hllc(12)");
- LEGACY_TYPE_MAP.put("hllc14", "hllc(14)");
- LEGACY_TYPE_MAP.put("hllc15", "hllc(15)");
- LEGACY_TYPE_MAP.put("hllc16", "hllc(16)");
-
- for (int i = 10; i <= 16; i++)
- HLLC_PRECISIONS.add(i);
- }
-
- private static final ConcurrentMap<DataType, DataType> CACHE = new ConcurrentHashMap<DataType, DataType>();
-
- public static DataType getInstance(String type) {
- if (type == null)
- return null;
-
- DataType dataType = new DataType(type);
- DataType cached = CACHE.get(dataType);
- if (cached == null) {
- CACHE.put(dataType, dataType);
- cached = dataType;
- }
- return cached;
- }
-
- // ============================================================================
-
- private String name;
- private int precision;
- private int scale;
-
- DataType(String datatype) {
- parseDataType(datatype);
- }
-
- private void parseDataType(String datatype) {
- datatype = datatype.trim().toLowerCase();
- datatype = replaceLegacy(datatype);
-
- Matcher m = TYPE_PATTERN.matcher(datatype);
- if (m.matches() == false)
- throw new IllegalArgumentException("bad data type -- " + datatype + ", does not match " + TYPE_PATTERN);
-
- name = replaceLegacy(m.group(1));
- precision = -1;
- scale = -1;
-
- String leftover = m.group(2);
- if (leftover != null) {
- String[] parts = leftover.split("\\s*,\\s*");
- for (int i = 0; i < parts.length; i++) {
- int n;
- try {
- n = Integer.parseInt(parts[i]);
- } catch (NumberFormatException e) {
- throw new IllegalArgumentException("bad data type -- " + datatype + ", precision/scale not numeric");
- }
- if (i == 0)
- precision = n;
- else if (i == 1)
- scale = n;
- else
- throw new IllegalArgumentException("bad data type -- " + datatype + ", too many precision/scale parts");
- }
- }
-
- // FIXME 256 for unknown string precision
- if ((name.equals("char") || name.equals("varchar")) && precision == -1) {
- precision = 256; // to save memory at frontend, e.g. tableau will
- // allocate memory according to this
- }
-
- // FIXME (19,4) for unknown decimal precision
- if ((name.equals("decimal") || name.equals("numeric")) && precision == -1) {
- precision = 19;
- scale = 4;
- }
-
- if (isHLLC() && HLLC_PRECISIONS.contains(precision) == false)
- throw new IllegalArgumentException("HLLC precision must be one of " + HLLC_PRECISIONS);
- }
-
- private String replaceLegacy(String str) {
- String replace = LEGACY_TYPE_MAP.get(str);
- return replace == null ? str : replace;
- }
-
- public int getSpaceEstimate() {
- if (isTinyInt()) {
- return 1;
- } else if (isSmallInt()) {
- return 2;
- } else if (isInt()) {
- return 4;
- } else if (isBigInt()) {
- return 8;
- } else if (isFloat()) {
- return 4;
- } else if (isDouble()) {
- return 8;
- } else if (isDecimal()) {
- return 8;
- } else if (isHLLC()) {
- return 1 << precision;
- }
- throw new IllegalStateException("The return type : " + name + " is not recognized;");
- }
-
- public boolean isStringFamily() {
- return STRING_FAMILY.contains(name);
- }
-
- public boolean isIntegerFamily() {
- return INTEGER_FAMILY.contains(name);
- }
-
- public boolean isNumberFamily() {
- return NUMBER_FAMILY.contains(name);
- }
-
- public boolean isDateTimeFamily() {
- return DATETIME_FAMILY.contains(name);
- }
-
- public boolean isTinyInt() {
- return name.equals("tinyint");
- }
-
- public boolean isSmallInt() {
- return name.equals("smallint");
- }
-
- public boolean isInt() {
- return name.equals("integer");
- }
-
- public boolean isBigInt() {
- return name.equals("bigint");
- }
-
- public boolean isFloat() {
- return name.equals("float");
- }
-
- public boolean isDouble() {
- return name.equals("double");
- }
-
- public boolean isDecimal() {
- return name.equals("decimal");
- }
-
- public boolean isHLLC() {
- return name.equals("hllc");
- }
-
- public String getName() {
- return name;
- }
-
- public int getPrecision() {
- return precision;
- }
-
- public int getScale() {
- return scale;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + precision;
- result = prime * result + scale;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- DataType other = (DataType) obj;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (precision != other.precision)
- return false;
- if (scale != other.scale)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- if (precision < 0 && scale < 0)
- return name;
- else if (scale < 0)
- return name + "(" + precision + ")";
- else
- return name + "(" + precision + "," + scale + ")";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/schema/DatabaseDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/schema/DatabaseDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/schema/DatabaseDesc.java
deleted file mode 100644
index 8183264..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/schema/DatabaseDesc.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.model.schema;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-
-/**
- * @author xjiang
- */
-public class DatabaseDesc {
- private String name;
-
- /**
- * @return the name
- */
- public String getName() {
- return name.toUpperCase();
- }
-
- /**
- * @param name
- * the name to set
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
-
- return "DatabaseDesc [name=" + name + "]";
- }
-
- public static HashMap<String, Integer> extractDatabaseOccurenceCounts(List<TableDesc> tables) {
- HashMap<String, Integer> databaseCounts = new HashMap<String, Integer>();
- for (TableDesc tableDesc : tables) {
- String databaseName = tableDesc.getDatabase();
- Integer counter = databaseCounts.get(databaseName);
- if (counter != null)
- databaseCounts.put(databaseName, counter + 1);
- else
- databaseCounts.put(databaseName, 1);
- }
- return databaseCounts;
- }
-
- public static HashSet<String> extractDatabaseNames(List<TableDesc> tables) {
- HashSet<String> databaseNames = new HashSet<String>();
- for (TableDesc tableDesc : tables) {
- databaseNames.add(tableDesc.getDatabase());
- }
- return databaseNames;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/model/schema/TableDesc.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/model/schema/TableDesc.java b/metadata/src/main/java/com/kylinolap/metadata/model/schema/TableDesc.java
deleted file mode 100644
index e10fe19..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/model/schema/TableDesc.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata.model.schema;
-
-import java.util.Arrays;
-import java.util.Comparator;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.kylinolap.common.persistence.ResourceStore;
-import com.kylinolap.common.persistence.RootPersistentEntity;
-
-/**
- * Table Metadata from Source. All name should be uppercase.
- * <p/>
- * User: lukhan Date: 10/15/13 Time: 9:06 AM To change this template use File |
- * Settings | File Templates.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class TableDesc extends RootPersistentEntity {
- @JsonProperty("name")
- private String name;
- @JsonProperty("columns")
- private ColumnDesc[] columns;
-
- private DatabaseDesc database;
-
- public ColumnDesc findColumnByName(String name) {
- for (ColumnDesc c : columns) {
- // return first matched column
- if (name.equalsIgnoreCase(c.getName())) {
- return c;
- }
- }
- return null;
- }
-
- public String getResourcePath() {
- return ResourceStore.TABLE_RESOURCE_ROOT + "/" + name + ".json";
- }
-
- // ============================================================================
-
- public String getName() {
- return this.name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- @JsonProperty("database")
- public String getDatabase() {
- if (database == null) {
- return "DEFAULT";
- }
- return database.getName();
- }
-
- @JsonProperty("database")
- public void setDatabase(String database) {
- this.database = new DatabaseDesc();
- this.database.setName(database);
- }
-
- public ColumnDesc[] getColumns() {
- return columns;
- }
-
- public void setColumns(ColumnDesc[] columns) {
- this.columns = columns;
- }
-
- public int getMaxColumnIndex() {
- int max = -1;
- for (ColumnDesc col : columns) {
- int idx = col.getZeroBasedIndex();
- max = Math.max(max, idx);
- }
- return max;
- }
-
- public int getColumnCount() {
- return getMaxColumnIndex() + 1;
- }
-
- public void init() {
- if (name != null)
- name = name.toUpperCase();
-
- if (getDatabase() != null)
- setDatabase(getDatabase().toUpperCase());
-
- if (columns != null) {
- Arrays.sort(columns, new Comparator<ColumnDesc>() {
- @Override
- public int compare(ColumnDesc col1, ColumnDesc col2) {
- Integer id1 = Integer.parseInt(col1.getId());
- Integer id2 = Integer.parseInt(col2.getId());
- return id1.compareTo(id2);
- }
- });
-
- for (ColumnDesc col : columns) {
- col.init(this);
- }
- }
- }
-
- @Override
- public String toString() {
- return "TableDesc [name=" + name + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java b/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java
deleted file mode 100644
index 3e4cca3..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/tool/HiveSourceTableLoader.java
+++ /dev/null
@@ -1,251 +0,0 @@
-package com.kylinolap.metadata.tool;
-
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import com.kylinolap.metadata.MetadataManager;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.common.persistence.ResourceTool;
-import com.kylinolap.common.util.CliCommandExecutor;
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.metadata.model.schema.ColumnDesc;
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * Management class to sync hive table metadata with command See main method for
- * how to use the class
- *
- * @author jianliu
- */
-public class HiveSourceTableLoader {
- private static final Logger logger = LoggerFactory.getLogger(HiveSourceTableLoader.class);
-
- public static final String OUTPUT_SURFIX = "json";
- public static final String TABLE_FOLDER_NAME = "table";
- public static final String TABLE_EXD_FOLDER_NAME = "table_exd";
-
- public static Set<String> reloadHiveTables(String[] hiveTables, KylinConfig config) throws IOException {
- Map<String, Set<String>> db2tables = Maps.newHashMap();
- for (String table : hiveTables) {
- int cut = table.indexOf('.');
- String database = cut >= 0 ? table.substring(0, cut).trim() : "DEFAULT";
- String tableName = cut >= 0 ? table.substring(cut + 1).trim() : table.trim();
- Set<String> set = db2tables.get(database);
- if (set == null) {
- set = Sets.newHashSet();
- db2tables.put(database, set);
- }
- set.add(tableName);
- }
-
- // metadata tmp dir
- File metaTmpDir = File.createTempFile("meta_tmp", null);
- metaTmpDir.delete();
- metaTmpDir.mkdirs();
-
- for (String database: db2tables.keySet()) {
- for (String table: db2tables.get(database)) {
- TableDesc tableDesc = MetadataManager.getInstance(config).getTableDesc(table);
- if (tableDesc == null) {
- continue;
- }
- if (tableDesc.getDatabase().equalsIgnoreCase(database)) {
- continue;
- } else {
- throw new UnsupportedOperationException(String.format("there is already a table[%s] in database[%s]", tableDesc.getName(), tableDesc.getDatabase()));
- }
- }
- }
-
- // extract from hive
- Set<String> loadedTables = Sets.newHashSet();
- for (String database : db2tables.keySet()) {
- List<String> loaded = extractHiveTables(database, db2tables.get(database), metaTmpDir, config);
- loadedTables.addAll(loaded);
- }
-
- // save loaded tables
- ResourceTool.copy(KylinConfig.createInstanceFromUri(metaTmpDir.getAbsolutePath()), config);
-
- return loadedTables;
- }
-
- private static List<String> extractHiveTables(String database, Set<String> tables, File metaTmpDir, KylinConfig config) throws IOException {
- StringBuilder cmd = new StringBuilder();
- cmd.append("hive -e \"");
- if (StringUtils.isEmpty(database) == false) {
- cmd.append("use " + database + "; ");
- }
- for (String table : tables) {
- cmd.append("show table extended like " + table + "; ");
- }
- cmd.append("\"");
-
- CliCommandExecutor cmdExec = config.getCliCommandExecutor();
- String output = cmdExec.execute(cmd.toString());
-
- return extractTableDescFromHiveOutput(database, output, metaTmpDir);
- }
-
- private static List<String> extractTableDescFromHiveOutput(String database, String hiveOutput, File metaTmpDir) throws IOException {
- BufferedReader reader = new BufferedReader(new StringReader(hiveOutput));
- try {
- return extractTables(database, reader, metaTmpDir);
- } finally {
- IOUtils.closeQuietly(reader);
- }
- }
-
- private static List<String> extractTables(String database, BufferedReader reader, File metaTmpDir) throws IOException {
-
- File tableDescDir = new File(metaTmpDir, TABLE_FOLDER_NAME);
- File tableExdDir = new File(metaTmpDir, TABLE_EXD_FOLDER_NAME);
- mkdirs(tableDescDir);
- mkdirs(tableExdDir);
-
- List<TableDesc> tableDescList = new ArrayList<TableDesc>();
- List<Map<String, String>> tableAttrsList = new ArrayList<Map<String, String>>();
- getTables(database, reader, tableDescList, tableAttrsList);
-
- List<String> loadedTables = Lists.newArrayList();
-
- for (TableDesc table : tableDescList) {
- File file = new File(tableDescDir, table.getName().toUpperCase() + "." + OUTPUT_SURFIX);
- JsonUtil.writeValueIndent(new FileOutputStream(file), table);
- loadedTables.add(table.getDatabase() + "." + table.getName());
- }
-
- for (Map<String, String> tableAttrs : tableAttrsList) {
- File file = new File(tableExdDir, tableAttrs.get("tableName").toUpperCase() + "." + OUTPUT_SURFIX);
- JsonUtil.writeValueIndent(new FileOutputStream(file), tableAttrs);
- }
- return loadedTables;
- }
-
- private static void mkdirs(File metaTmpDir) {
- if (!metaTmpDir.exists()) {
- if (!metaTmpDir.mkdirs()) {
- throw new IllegalArgumentException("Failed to create Output dir : " + metaTmpDir.getAbsolutePath());
- }
- }
- }
-
- private static void getTables(String database, BufferedReader reader, //
- List<TableDesc> tableDescList, List<Map<String, String>> tableAttrsList) throws IOException {
-
- Map<String, String> tableAttrs = new HashMap<String, String>();
- TableDesc tableDesc = new TableDesc();
- String line;
- boolean hit = false;
-
- while ((line = reader.readLine()) != null) {
- logger.info(line);
- int i = line.indexOf(":");
- if (i == -1) {
- continue;
- }
- String key = line.substring(0, i);
- String value = line.substring(i + 1, line.length());
- if (key.equals("tableName")) {// Create a new table object
- hit = true;
- tableAttrs = new HashMap<String, String>();
- tableAttrsList.add(tableAttrs);
- tableDesc = new TableDesc();
- tableDescList.add(tableDesc);
- }
-
- if (!hit) {
- continue;
- }
-
- if (line.startsWith("columns")) {// geneate source table metadata
- String tname = tableAttrs.get("tableName");
-
- tableDesc.setDatabase(database.toUpperCase());
- tableDesc.setName(tname.toUpperCase());
- tableDesc.setUuid(UUID.randomUUID().toString());
- addColumns(tableDesc, value);
- }
- tableAttrs.put(key, value);
- if (key.equals("lastUpdateTime")) {
- hit = false;
- }
- }
-
- }
-
- private static void addColumns(TableDesc sTable, String value) {
- List<ColumnDesc> columns = new ArrayList<ColumnDesc>();
- int i1 = value.indexOf("{");
- int i2 = value.indexOf("}");
- if (i1 < 0 || i2 < 0 || i1 > i2) {
- return;
- }
- String temp = value.substring(i1 + 1, i2);
- String[] strArr = temp.split(", ");
- for (int i = 0; i < strArr.length; i++) {
- String t1 = strArr[i].trim();
- int pos = t1.indexOf(" ");
- String colType = t1.substring(0, pos).trim();
- String colName = t1.substring(pos).trim();
- ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(colName.toUpperCase());
- cdesc.setDatatype(convertType(colType));
- cdesc.setId(String.valueOf(i + 1));
- columns.add(cdesc);
- }
- sTable.setColumns(columns.toArray(new ColumnDesc[0]));
- }
-
- private static String convertType(String colType) {
- if ("i32".equals(colType)) {
- return "int";
- } else if ("i64".equals(colType)) {
- return "bigint";
- } else if ("i16".equals(colType)) {
- return "smallint";
- } else if ("byte".equals(colType)) {
- return "tinyint";
- } else if ("bool".equals(colType))
- return "boolean";
- return colType;
- }
-
- /**
- */
- public static void main(String[] args) {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/CubeMetadataValidator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/CubeMetadataValidator.java b/metadata/src/main/java/com/kylinolap/metadata/validation/CubeMetadataValidator.java
deleted file mode 100644
index 7ee2f97..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/CubeMetadataValidator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata.validation;
-
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.validation.ValidateContext.Result;
-import com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule;
-import com.kylinolap.metadata.validation.rule.FunctionRule;
-import com.kylinolap.metadata.validation.rule.MandatoryColumnRule;
-import com.kylinolap.metadata.validation.rule.RowKeyAttrRule;
-
-/**
- * For cube metadata validator
- *
- * @author jianliu
- *
- */
-public class CubeMetadataValidator {
- @SuppressWarnings("unchecked")
- private IValidatorRule<CubeDesc>[] rules = new IValidatorRule[] { new FunctionRule(), new AggregationGroupSizeRule(), new MandatoryColumnRule(), new RowKeyAttrRule() };
-
- public ValidateContext validate(CubeDesc cube) {
- return validate(cube, false);
- }
-
- /**
- * @param cubeDesc
- * @param inject
- * inject error into cube desc
- * @return
- */
- public ValidateContext validate(CubeDesc cube, boolean inject) {
- ValidateContext context = new ValidateContext();
- for (int i = 0; i < rules.length; i++) {
- IValidatorRule<CubeDesc> rule = rules[i];
- rule.validate(cube, context);
- }
- if (inject) {
- injectResult(cube, context);
- }
- return context;
- }
-
- /**
- *
- * Inject errors info into cubeDesc
- *
- * @param cubeDesc
- * @param context
- */
- public void injectResult(CubeDesc cubeDesc, ValidateContext context) {
- Result[] results = context.getResults();
- for (int i = 0; i < results.length; i++) {
- Result result = results[i];
- cubeDesc.addError(result.getLevel() + " : " + result.getMessage(), true);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/IValidatorRule.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/IValidatorRule.java b/metadata/src/main/java/com/kylinolap/metadata/validation/IValidatorRule.java
deleted file mode 100644
index 1091775..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/IValidatorRule.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation;
-
-import com.kylinolap.metadata.validation.rule.IKylinValidationConstances;
-
-/**
- * @author jianliu
- *
- */
-public interface IValidatorRule<T> extends IKylinValidationConstances {
-
- public void validate(T element, ValidateContext context);
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/ResultLevel.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/ResultLevel.java b/metadata/src/main/java/com/kylinolap/metadata/validation/ResultLevel.java
deleted file mode 100644
index 32af9d7..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/ResultLevel.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation;
-
-/**
- * Validation result level
- *
- * @author jianliu
- *
- */
-public enum ResultLevel {
- ERROR("ERROR"), WARN("WARN");
- private String level;
-
- ResultLevel(String level) {
- this.level = level;
- }
-
- public String toString() {
- return level;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/SourceTableMetadataValidator.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/SourceTableMetadataValidator.java b/metadata/src/main/java/com/kylinolap/metadata/validation/SourceTableMetadataValidator.java
deleted file mode 100644
index deff50b..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/SourceTableMetadataValidator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata.validation;
-
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * Validate Table metadata from source.
- * <p/>
- * User: lukhan Date: 12/2/13 Time: 10:45 AM To change this template use File |
- * Settings | File Templates.
- */
-public class SourceTableMetadataValidator {
-
- public static boolean validate(TableDesc table) {
- // table.get
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/ValidateContext.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/ValidateContext.java b/metadata/src/main/java/com/kylinolap/metadata/validation/ValidateContext.java
deleted file mode 100644
index bd27244..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/ValidateContext.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation;
-
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Context. Supply all dependent objects for validator
- *
- * @author jianliu
- *
- */
-public class ValidateContext {
- private List<Result> results = new ArrayList<ValidateContext.Result>();
-
- public void addResult(ResultLevel level, String message) {
- results.add(new Result(level, message));
- }
-
- public void addResult(Result result) {
- results.add(result);
- }
-
- public class Result {
- private ResultLevel level;
- private String message;
-
- /**
- * @param level
- * @param message
- */
- public Result(ResultLevel level, String message) {
- this.level = level;
- this.message = message;
- }
-
- /**
- * @return the level
- */
- public ResultLevel getLevel() {
- return level;
- }
-
- /**
- * @return the message
- */
- public String getMessage() {
- return message;
- }
- }
-
- /**
- * Get validation result
- *
- * @return
- */
- public Result[] getResults() {
- Result[] rs = new Result[0];
- rs = results.toArray(rs);
- return rs;
- }
-
- /**
- *
- */
- public void print(PrintStream out) {
- if (results.isEmpty()) {
- out.print("The element is perfect.");
- }
- Iterator<Result> it = results.iterator();
- while (it.hasNext()) {
- Result result = it.next();
- out.println(result.level + " : " + result.message);
- }
- }
-
- /**
- * @return if there is not validation errors
- */
- public boolean ifPass() {
- return results.isEmpty();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRule.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRule.java b/metadata/src/main/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRule.java
deleted file mode 100644
index 8ab5a7e..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRule.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ResultLevel;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * Rule to validate: 1. The aggregationGroup size must be less than 20
- *
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , com.kylinolap.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- innerValidateMaxSize(cube, context);
- }
-
- /**
- * @param cube
- * @param context
- */
- private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
- int maxSize = getMaxAgrGroupSize();
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- if (group.length >= maxSize) {
- context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
- }
- }
- }
-
- protected int getMaxAgrGroupSize() {
- String size = KylinConfig.getInstanceFromEnv().getProperty(KEY_MAX_AGR_GROUP_SIZE, String.valueOf(DEFAULT_MAX_AGR_GROUP_SIZE));
- return Integer.parseInt(size);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/rule/FunctionRule.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/FunctionRule.java b/metadata/src/main/java/com/kylinolap/metadata/validation/rule/FunctionRule.java
deleted file mode 100644
index 4d2543e..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/FunctionRule.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.kylinolap.common.KylinConfig;
-import com.kylinolap.metadata.MetadataManager;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.model.cube.FunctionDesc;
-import com.kylinolap.metadata.model.cube.MeasureDesc;
-import com.kylinolap.metadata.model.cube.ParameterDesc;
-import com.kylinolap.metadata.model.schema.ColumnDesc;
-import com.kylinolap.metadata.model.schema.DataType;
-import com.kylinolap.metadata.model.schema.TableDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ResultLevel;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * Validate function parameter. Ticket:
- * https://github.scm.corp.ebay.com/Kylin/Kylin/issues/268
- * <p/>
- * if type is column, check values are valid fact table columns if type is
- * constant, the value only can be numberic
- * <p/>
- * the return type only can be int/bigint/long/double/decimal
- *
- * @author jianliu
- */
-public class FunctionRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , com.kylinolap.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- List<MeasureDesc> measures = cube.getMeasures();
-
- List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>();
-
- Iterator<MeasureDesc> it = measures.iterator();
- while (it.hasNext()) {
- MeasureDesc measure = it.next();
- FunctionDesc func = measure.getFunction();
- ParameterDesc parameter = func.getParameter();
- if (parameter == null) {
- context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
- return;
- }
-
- String type = func.getParameter().getType();
- String value = func.getParameter().getValue();
- if (StringUtils.isEmpty(type)) {
- context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
- return;
- }
- if (StringUtils.isEmpty(value)) {
- context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
- return;
- }
- if (StringUtils.isEmpty(func.getReturnType())) {
- context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
- return;
- }
-
- if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
- validateColumnParameter(context, cube, value);
- } else if (StringUtils.equals(FunctionDesc.PARAMTER_TYPE_CONSTANT, type)) {
- validateCostantParameter(context, cube, value);
- }
- validateReturnType(context, cube, func);
-
- if (func.isCount())
- countFuncs.add(func);
- }
-
- if (countFuncs.size() != 1) {
- context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs);
- }
- }
-
- private void validateReturnType(ValidateContext context, CubeDesc cube, FunctionDesc funcDesc) {
-
- String func = funcDesc.getExpression();
- DataType rtype = funcDesc.getReturnDataType();
-
- if (funcDesc.isCount()) {
- if (rtype.isIntegerFamily() == false) {
- context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.INTEGER_FAMILY);
- }
- } else if (funcDesc.isCountDistinct()) {
- if (rtype.isHLLC() == false && funcDesc.isHolisticCountDistinct() == false) {
- context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be hllc(10), hllc(12) etc.");
- }
- } else if (funcDesc.isMax() || funcDesc.isMin() || funcDesc.isSum()) {
- if (rtype.isNumberFamily() == false) {
- context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
- }
- } else {
- if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) {
- context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]");
- }
- }
-
- }
-
- /**
- * @param context
- * @param cube
- * @param value
- */
- private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) {
- try {
- Integer.parseInt(value);
- } catch (Exception e) {
- context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value);
- }
- }
-
- /**
- * @param context
- * @param cube
- * @param value
- */
- private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
- String factTable = cube.getFactTable();
- if (StringUtils.isEmpty(factTable)) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
- return;
- }
- TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
- if (table == null) {
- context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
- return;
- }
- // Prepare column set
- Set<String> set = new HashSet<String>();
- ColumnDesc[] cdesc = table.getColumns();
- for (int i = 0; i < cdesc.length; i++) {
- ColumnDesc columnDesc = cdesc[i];
- set.add(columnDesc.getName());
- }
-
- String[] items = value.split(",");
- for (int i = 0; i < items.length; i++) {
- String item = items[i].trim();
- if (StringUtils.isEmpty(item)) {
- continue;
- }
- if (!set.contains(item)) {
- context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/rule/IKylinValidationConstances.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/IKylinValidationConstances.java b/metadata/src/main/java/com/kylinolap/metadata/validation/rule/IKylinValidationConstances.java
deleted file mode 100644
index bc1bbc6..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/IKylinValidationConstances.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import com.kylinolap.metadata.MetadataConstances;
-
-/**
- * @author jianliu
- *
- */
-public interface IKylinValidationConstances extends MetadataConstances {
-
- public static final int DEFAULT_MAX_AGR_GROUP_SIZE = 20;
- public static final String KEY_MAX_AGR_GROUP_SIZE = "rule_max.arggregation.group.size";
- public static final String KEY_IGNORE_UNKNOWN_FUNC = "rule_ignore_unknown_func";
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRule.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRule.java b/metadata/src/main/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRule.java
deleted file mode 100644
index 2e2d43e..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRule.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang.ArrayUtils;
-
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.model.cube.RowKeyColDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ResultLevel;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * Validate that mandatory column must NOT appear in aggregation group.
- *
- * @author jianliu
- *
- */
-public class MandatoryColumnRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , com.kylinolap.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- Set<String> mands = new HashSet<String>();
- RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
- if (cols == null || cols.length == 0) {
- return;
- }
- for (int i = 0; i < cols.length; i++) {
- RowKeyColDesc rowKeyColDesc = cols[i];
- if (rowKeyColDesc.isMandatory()) {
- mands.add(rowKeyColDesc.getColumn());
- }
- }
- if (mands.isEmpty()) {
- return;
- }
- String[][] groups = cube.getRowkey().getAggregationGroups();
- for (int i = 0; i < groups.length; i++) {
- String[] group = groups[i];
- for (int j = 0; j < group.length; j++) {
- String col = group[j];
- if (mands.contains(col)) {
- context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
- }
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/main/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRule.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRule.java b/metadata/src/main/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRule.java
deleted file mode 100644
index 5253cea..0000000
--- a/metadata/src/main/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRule.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.model.cube.RowKeyColDesc;
-import com.kylinolap.metadata.model.cube.RowKeyDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ResultLevel;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * Validate that only one of "length" and "dictionary" appears on rowkey_column
- *
- * @author jianliu
- *
- */
-public class RowKeyAttrRule implements IValidatorRule<CubeDesc> {
-
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.IValidatorRule#validate(java.lang.Object
- * , com.kylinolap.metadata.validation.ValidateContext)
- */
- @Override
- public void validate(CubeDesc cube, ValidateContext context) {
- RowKeyDesc row = cube.getRowkey();
- if (row == null) {
- context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
- return;
- }
-
- RowKeyColDesc[] rcd = row.getRowKeyColumns();
- if (rcd == null) {
- context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
- return;
- }
- if(rcd.length == 0){
- context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
- return;
- }
-
- for (int i = 0; i < rcd.length; i++) {
- RowKeyColDesc rd = rcd[i];
- if (rd.getLength() != 0 && !StringUtils.isEmpty(rd.getDictionary())) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
- }
- if (rd.getLength() == 0 && StringUtils.isEmpty(rd.getDictionary())) {
- context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java b/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java
deleted file mode 100644
index 1320b73..0000000
--- a/metadata/src/test/java/com/kylinolap/metadata/CubeDescTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-
-/**
- * @author yangli9
- */
-public class CubeDescTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testSerialize() throws Exception {
- CubeDesc desc = MetadataManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
- String str = JsonUtil.writeValueAsIndentString(desc);
- System.out.println(str);
- @SuppressWarnings("unused")
- CubeDesc desc2 = JsonUtil.readValue(str, CubeDesc.class);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java b/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java
deleted file mode 100644
index a559ac7..0000000
--- a/metadata/src/test/java/com/kylinolap/metadata/MetadataManagerTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.kylinolap.metadata;
-
-import java.util.List;
-
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.LocalFileMetadataTestCase;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.model.schema.TableDesc;
-
-/**
- * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 2:38 PM To
- * change this template use File | Settings | File Templates.
- */
-public class MetadataManagerTest extends LocalFileMetadataTestCase {
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
- }
-
- @After
- public void after() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test
- public void testListAllTables() throws Exception {
- List<TableDesc> tables = MetadataManager.getInstance(this.getTestConfig()).listAllTables();
- Assert.assertNotNull(tables);
- Assert.assertTrue(tables.size() > 0);
- }
-
- @Test
- public void testFindTableByName() throws Exception {
- TableDesc table = MetadataManager.getInstance(this.getTestConfig()).getTableDesc("TEST_CAL_DT");
- Assert.assertNotNull(table);
- Assert.assertEquals("TEST_CAL_DT", table.getName());
- }
-
- @Test
- public void testGetInstance() throws Exception {
-
- Assert.assertNotNull(MetadataManager.getInstance(this.getTestConfig()));
- Assert.assertNotNull(MetadataManager.getInstance(this.getTestConfig()).listAllTables());
- Assert.assertTrue(MetadataManager.getInstance(this.getTestConfig()).listAllTables().size() > 0);
- }
-
- @Test
- public void testGetCubeDesc() throws Exception {
- CubeDesc cubeDesc = MetadataManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_with_slr_desc");
- Assert.assertNotNull(cubeDesc);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java
deleted file mode 100644
index 356c6e7..0000000
--- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/AggregationGroupSizeRuleTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import static org.junit.Assert.*;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * @author jianliu
- *
- */
-public class AggregationGroupSizeRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST2_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 3;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 2);
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[0].getMessage().startsWith("Length of the number"));
- assertTrue("Failed to validate aggragation group error", vContext.getResults()[1].getMessage().startsWith("Length of the number"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[2].getMessage()
- // .startsWith("Hierachy column"));
- }
-
- @Test
- public void testAggColumnSize() {
- AggregationGroupSizeRule rule = new AggregationGroupSizeRule() {
- /*
- * (non-Javadoc)
- *
- * @see
- * com.kylinolap.metadata.validation.rule.AggregationGroupSizeRule
- * #getMaxAgrGroupSize()
- */
- @Override
- protected int getMaxAgrGroupSize() {
- return 20;
- }
- };
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertEquals("Failed to validate aggragation group error", vContext.getResults().length, 0);
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Aggregation group"));
- // assertTrue("Failed to validate aggragation group error",
- // vContext.getResults()[0].getMessage()
- // .startsWith("Hierachy column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java
deleted file mode 100644
index a78bfe4..0000000
--- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/MandatoryColumnRuleTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import static org.junit.Assert.*;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * @author jianliu
- *
- */
-public class MandatoryColumnRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST1_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new MandatoryColumnRule();
- rule.validate(cube, vContext);
- assertTrue("Failed to validate mandatory error", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("mandatory column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java
----------------------------------------------------------------------
diff --git a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java b/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java
deleted file mode 100644
index 4f9ebe3..0000000
--- a/metadata/src/test/java/com/kylinolap/metadata/validation/rule/RowKeyAttrRuleTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2013-2014 eBay Software Foundation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.kylinolap.metadata.validation.rule;
-
-import static org.junit.Assert.*;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import com.kylinolap.common.util.JsonUtil;
-import com.kylinolap.metadata.model.cube.CubeDesc;
-import com.kylinolap.metadata.validation.IValidatorRule;
-import com.kylinolap.metadata.validation.ValidateContext;
-
-/**
- * @author jianliu
- *
- */
-public class RowKeyAttrRuleTest {
-
- private CubeDesc cube;
- private ValidateContext vContext = new ValidateContext();
-
- /**
- * @throws java.lang.Exception
- */
- @Before
- public void setUp() throws Exception {
- CubeDesc desc2 = JsonUtil.readValue(getClass().getClassLoader().getResourceAsStream("data/TEST3_desc.json"), CubeDesc.class);
- this.cube = desc2;
-
- }
-
- @Test
- public void testOneMandatoryColumn() {
- IValidatorRule<CubeDesc> rule = new RowKeyAttrRule();
- rule.validate(cube, vContext);
- vContext.print(System.out);
- assertTrue("Failed to validate rowkey", vContext.getResults().length == 1);
- assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a4fd4268/metadata/src/test/resources/com/kylinolap/metadata/kylin_env.properties
----------------------------------------------------------------------
diff --git a/metadata/src/test/resources/com/kylinolap/metadata/kylin_env.properties b/metadata/src/test/resources/com/kylinolap/metadata/kylin_env.properties
deleted file mode 100644
index b9acf90..0000000
--- a/metadata/src/test/resources/com/kylinolap/metadata/kylin_env.properties
+++ /dev/null
@@ -1,5 +0,0 @@
-PROD=/etc/kylin
-QA=/etc/kylin
-DEV=/etc/kylin
-
-YADEV=hbase:yadesk00:2181:/hbase-unsecure
\ No newline at end of file