You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/03/17 03:10:33 UTC
[2/7] kylin git commit: KYLIN-1495 Metadata upgrade tool from 1.0~1.3
to 1.5
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeDesc.java
deleted file mode 100644
index 9467e9a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/CubeDesc.java
+++ /dev/null
@@ -1,825 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.common.util.Array;
-import org.apache.kylin.common.util.CaseInsensitiveStringMap;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.measure.MeasureType;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.IEngineAware;
-import org.apache.kylin.metadata.model.IStorageAware;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- */
-@SuppressWarnings("serial")
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeDesc extends RootPersistentEntity {
-
- public static enum DeriveType {
- LOOKUP, PK_FK
- }
-
- public static class DeriveInfo {
- public DeriveType type;
- public DimensionDesc dimension;
- public TblColRef[] columns;
- public boolean isOneToOne; // only used when ref from derived to host
-
- DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) {
- this.type = type;
- this.dimension = dimension;
- this.columns = columns;
- this.isOneToOne = isOneToOne;
- }
-
- @Override
- public String toString() {
- return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]";
- }
-
- }
-
- private KylinConfig config;
- private DataModelDesc model;
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("model_name")
- private String modelName;
- @JsonProperty("description")
- private String description;
- @JsonProperty("null_string")
- private String[] nullStrings;
- @JsonProperty("dimensions")
- private List<DimensionDesc> dimensions;
- @JsonProperty("measures")
- private List<MeasureDesc> measures;
- @JsonProperty("rowkey")
- private RowKeyDesc rowkey;
- @JsonProperty("hbase_mapping")
- private HBaseMappingDesc hbaseMapping;
- @JsonProperty("signature")
- private String signature;
- @JsonProperty("notify_list")
- private List<String> notifyList;
- @JsonProperty("status_need_notify")
- private List<String> statusNeedNotify = Collections.emptyList();
- @JsonProperty("auto_merge_time_ranges")
- private long[] autoMergeTimeRanges;
- @JsonProperty("retention_range")
- private long retentionRange = 0;
-
- @JsonProperty("engine_type")
- private int engineType = IEngineAware.ID_MR_V1;
- @JsonProperty("storage_type")
- private int storageType = IStorageAware.ID_HBASE;
-
- private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
- private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
- private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>();
-
- private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap();
- private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap();
-
- public boolean isEnableSharding() {
- //in the future may extend to other storage that is shard-able
- return storageType == IStorageAware.ID_SHARDED_HBASE;
- }
-
- /**
- * Error messages during resolving json metadata
- */
- private List<String> errors = new ArrayList<String>();
-
- /**
- * @return all columns this cube can support, including derived
- */
- public Set<TblColRef> listAllColumns() {
- return allColumns;
- }
-
- /**
- * @return dimension columns including derived, BUT NOT measures
- */
- public Set<TblColRef> listDimensionColumnsIncludingDerived() {
- return dimensionColumns;
- }
-
- /**
- * @return dimension columns excluding derived and measures
- */
- public List<TblColRef> listDimensionColumnsExcludingDerived() {
- List<TblColRef> result = new ArrayList<TblColRef>();
- for (TblColRef col : dimensionColumns) {
- if (isDerived(col) == false)
- result.add(col);
- }
- return result;
- }
-
- /**
- * Find FunctionDesc by Full Expression.
- *
- * @return
- */
- public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) {
- for (MeasureDesc m : measures) {
- if (m.getFunction().equals(manualFunc))
- return m.getFunction();
- }
- return null;
- }
-
- public TblColRef findColumnRef(String table, String column) {
- Map<String, TblColRef> cols = columnMap.get(table);
- if (cols == null)
- return null;
- else
- return cols.get(column);
- }
-
- public DimensionDesc findDimensionByColumn(TblColRef col) {
- for (DimensionDesc dim : dimensions) {
- if (ArrayUtils.contains(dim.getColumnRefs(), col))
- return dim;
- }
- return null;
- }
-
- public DimensionDesc findDimensionByTable(String lookupTableName) {
- lookupTableName = lookupTableName.toUpperCase();
- for (DimensionDesc dim : dimensions)
- if (dim.getTable() != null && dim.getTable().equals(lookupTableName))
- return dim;
- return null;
- }
-
- public DimensionDesc findDimensionByName(String dimName) {
- dimName = dimName.toUpperCase();
- for (DimensionDesc dim : dimensions) {
- if (dimName.equals(dim.getName()))
- return dim;
- }
- return null;
- }
-
- /**
- * Get all functions from each measure.
- *
- * @return
- */
- public List<FunctionDesc> listAllFunctions() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
- for (MeasureDesc m : measures) {
- functions.add(m.getFunction());
- }
- return functions;
- }
-
- public boolean isDerived(TblColRef col) {
- return derivedToHostMap.containsKey(col);
- }
-
- public DeriveInfo getHostInfo(TblColRef derived) {
- return derivedToHostMap.get(derived);
- }
-
- public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) {
- Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>();
- for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) {
- Array<TblColRef> hostCols = entry.getKey();
- boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data));
- if (!hostOnRow)
- continue;
-
- List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>();
- for (DeriveInfo info : entry.getValue()) {
- if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has
- // any
- // wanted
- // columns?
- wantedInfo.add(info);
- }
-
- if (wantedInfo.size() > 0)
- result.put(hostCols, wantedInfo);
- }
- return result;
- }
-
- public String getResourcePath() {
- return getCubeDescResourcePath(name);
- }
-
- public static String getCubeDescResourcePath(String descName) {
- return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
- }
-
- // ============================================================================
-
- public HBaseMappingDesc getHBaseMapping() {
- return hbaseMapping;
- }
-
- public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getModelName() {
- return modelName;
- }
-
- public void setModelName(String modelName) {
- this.modelName = modelName;
- }
-
- public DataModelDesc getModel() {
- return model;
- }
-
- public void setModel(DataModelDesc model) {
- this.model = model;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public String getFactTable() {
- return model.getFactTable();
- }
-
- public TableDesc getFactTableDesc() {
- return model.getFactTableDesc();
- }
-
- public String[] getNullStrings() {
- return nullStrings;
- }
-
- public List<DimensionDesc> getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(List<DimensionDesc> dimensions) {
- this.dimensions = dimensions;
- }
-
- public List<MeasureDesc> getMeasures() {
- return measures;
- }
-
- public void setMeasures(List<MeasureDesc> measures) {
- this.measures = measures;
- }
-
- public RowKeyDesc getRowkey() {
- return rowkey;
- }
-
- public void setRowkey(RowKeyDesc rowkey) {
- this.rowkey = rowkey;
- }
-
- public String getSignature() {
- return signature;
- }
-
- public void setSignature(String signature) {
- this.signature = signature;
- }
-
- public List<String> getNotifyList() {
- return notifyList;
- }
-
- public void setNotifyList(List<String> notifyList) {
- this.notifyList = notifyList;
- }
-
- public List<String> getStatusNeedNotify() {
- return statusNeedNotify;
- }
-
- public void setStatusNeedNotify(List<String> statusNeedNotify) {
- this.statusNeedNotify = statusNeedNotify;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- CubeDesc cubeDesc = (CubeDesc) o;
-
- if (!name.equals(cubeDesc.name))
- return false;
- if (!getFactTable().equals(cubeDesc.getFactTable()))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = 0;
- result = 31 * result + name.hashCode();
- result = 31 * result + getFactTable().hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "CubeDesc [name=" + name + "]";
- }
-
- public boolean checkSignature() {
- if (StringUtils.isBlank(getSignature())) {
- return true;
- }
- return calculateSignature().equals(getSignature());
- }
-
- public String calculateSignature() {
- MessageDigest md = null;
- try {
- md = MessageDigest.getInstance("MD5");
- StringBuilder sigString = new StringBuilder();
- sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping));
-
- byte[] signature = md.digest(sigString.toString().getBytes());
- return new String(Base64.encodeBase64(signature));
- } catch (NoSuchAlgorithmException | JsonProcessingException e) {
- throw new RuntimeException("Failed to calculate signature");
- }
- }
-
- public Map<String, TblColRef> buildColumnNameAbbreviation() {
- Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>();
- for (TblColRef col : listDimensionColumnsExcludingDerived()) {
- r.put(col.getName(), col);
- }
- return r;
- }
-
- public void init(KylinConfig config, Map<String, TableDesc> tables) {
- this.errors.clear();
- this.config = config;
-
- if (this.modelName == null || this.modelName.length() == 0) {
- this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified.");
- }
-
- this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName);
-
- if (this.model == null) {
- this.addError("No data model found with name '" + modelName + "'.");
- }
-
- for (DimensionDesc dim : dimensions) {
- dim.init(this, tables);
- }
-
- sortDimAndMeasure();
- initDimensionColumns();
- initMeasureColumns();
-
- rowkey.init(this);
- if (hbaseMapping != null) {
- hbaseMapping.init(this);
- }
-
- initMeasureReferenceToColumnFamily();
-
- // check all dimension columns are presented on rowkey
- List<TblColRef> dimCols = listDimensionColumnsExcludingDerived();
- if (rowkey.getRowKeyColumns().length != dimCols.size()) {
- addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). ");
- }
- }
-
- private void initDimensionColumns() {
- for (DimensionDesc dim : dimensions) {
- JoinDesc join = dim.getJoin();
-
- // init dimension columns
- ArrayList<TblColRef> dimCols = Lists.newArrayList();
- String[] colStrs = dim.getColumn();
-
- // when column is omitted, special case
- if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) {
- for (TblColRef col : join.getForeignKeyColumns()) {
- dimCols.add(initDimensionColRef(col));
- }
- }
- // normal case
- else {
- if (colStrs == null || colStrs.length == 0)
- throw new IllegalStateException("Dimension column must not be blank " + dim);
-
- for (String colStr : colStrs) {
- dimCols.add(initDimensionColRef(dim, colStr));
- }
-
- // fill back column ref in hierarchy
- if (dim.isHierarchy()) {
- for (int i = 0; i < dimCols.size(); i++)
- dim.getHierarchy()[i].setColumnRef(dimCols.get(i));
- }
- }
-
- TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]);
- dim.setColumnRefs(dimColArray);
-
- // init derived columns
- TblColRef[] hostCols = dimColArray;
- if (dim.isDerived()) {
- String[] derived = dim.getDerived();
- String[][] split = splitDerivedColumnAndExtra(derived);
- String[] derivedNames = split[0];
- String[] derivedExtra = split[1];
- TblColRef[] derivedCols = new TblColRef[derivedNames.length];
- for (int i = 0; i < derivedNames.length; i++) {
- derivedCols[i] = initDimensionColRef(dim, derivedNames[i]);
- }
- initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra);
- }
-
- // PK-FK derive the other side
- if (join != null) {
- TblColRef[] fk = join.getForeignKeyColumns();
- TblColRef[] pk = join.getPrimaryKeyColumns();
-
- allColumns.addAll(Arrays.asList(fk));
- allColumns.addAll(Arrays.asList(pk));
- for (int i = 0; i < fk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, fk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(pk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
- /** disable this code as we don't need fk be derived from pk
- for (int i = 0; i < pk.length; i++) {
- int find = ArrayUtils.indexOf(hostCols, pk[i]);
- if (find >= 0) {
- TblColRef derivedCol = initDimensionColRef(fk[i]);
- initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol);
- }
- }
- */
- }
- }
- }
-
- private String[][] splitDerivedColumnAndExtra(String[] derived) {
- String[] cols = new String[derived.length];
- String[] extra = new String[derived.length];
- for (int i = 0; i < derived.length; i++) {
- String str = derived[i];
- int cut = str.indexOf(":");
- if (cut >= 0) {
- cols[i] = str.substring(0, cut);
- extra[i] = str.substring(cut + 1).trim();
- } else {
- cols[i] = str;
- extra[i] = "";
- }
- }
- return new String[][] { cols, extra };
- }
-
- private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) {
- initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null);
- }
-
- private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) {
- if (hostCols.length == 0 || derivedCols.length == 0)
- throw new IllegalStateException("host/derived columns must not be empty");
-
- // Although FK derives PK automatically, user unaware of this can declare PK as derived dimension explicitly.
- // In that case, derivedCols[] will contain a FK which is transformed from the PK by initDimensionColRef().
- // Must drop FK from derivedCols[] before continue.
- for (int i = 0; i < derivedCols.length; i++) {
- if (ArrayUtils.contains(hostCols, derivedCols[i])) {
- derivedCols = (TblColRef[]) ArrayUtils.remove(derivedCols, i);
- extra = (String[]) ArrayUtils.remove(extra, i);
- i--;
- }
- }
-
- Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols);
- List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray);
- if (infoList == null) {
- hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>());
- }
- infoList.add(new DeriveInfo(type, dimension, derivedCols, false));
-
- for (int i = 0; i < derivedCols.length; i++) {
- TblColRef derivedCol = derivedCols[i];
- boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1"));
- derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne));
- }
- }
-
- private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
- TableDesc table = dim.getTableDesc();
- ColumnDesc col = table.findColumnByName(colName);
- if (col == null)
- throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
-
- TblColRef ref = new TblColRef(col);
-
- // always use FK instead PK, FK could be shared by more than one lookup tables
- JoinDesc join = dim.getJoin();
- if (join != null) {
- int idx = ArrayUtils.indexOf(join.getPrimaryKeyColumns(), ref);
- if (idx >= 0) {
- ref = join.getForeignKeyColumns()[idx];
- }
- }
- return initDimensionColRef(ref);
- }
-
- private TblColRef initDimensionColRef(TblColRef ref) {
- TblColRef existing = findColumnRef(ref.getTable(), ref.getName());
- if (existing != null) {
- return existing;
- }
-
- allColumns.add(ref);
- dimensionColumns.add(ref);
-
- Map<String, TblColRef> cols = columnMap.get(ref.getTable());
- if (cols == null) {
- columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>());
- }
- cols.put(ref.getName(), ref);
- return ref;
- }
-
- private void initMeasureColumns() {
- if (measures == null || measures.isEmpty()) {
- return;
- }
-
- TableDesc factTable = getFactTableDesc();
- for (MeasureDesc m : measures) {
- m.setName(m.getName().toUpperCase());
-
- if (m.getDependentMeasureRef() != null) {
- m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
- }
-
- FunctionDesc func = m.getFunction();
- func.init(factTable, null);
- allColumns.addAll(func.getParameter().getColRefs());
-
- func.getMeasureType().validate(func);
- }
- }
-
- private void initMeasureReferenceToColumnFamily() {
- if (measures == null || measures.size() == 0)
- return;
-
- Map<String, MeasureDesc> measureLookup = new HashMap<String, MeasureDesc>();
- for (MeasureDesc m : measures)
- measureLookup.put(m.getName(), m);
- Map<String, Integer> measureIndexLookup = new HashMap<String, Integer>();
- for (int i = 0; i < measures.size(); i++)
- measureIndexLookup.put(measures.get(i).getName(), i);
-
- for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
- for (HBaseColumnDesc c : cf.getColumns()) {
- String[] colMeasureRefs = c.getMeasureRefs();
- MeasureDesc[] measureDescs = new MeasureDesc[colMeasureRefs.length];
- int[] measureIndex = new int[colMeasureRefs.length];
- for (int i = 0; i < colMeasureRefs.length; i++) {
- measureDescs[i] = measureLookup.get(colMeasureRefs[i]);
- measureIndex[i] = measureIndexLookup.get(colMeasureRefs[i]);
- }
- c.setMeasures(measureDescs);
- c.setMeasureIndex(measureIndex);
- c.setColumnFamilyName(cf.getName());
- }
- }
- }
-
- private void sortDimAndMeasure() {
- sortDimensionsByID();
- sortMeasuresByID();
- for (DimensionDesc dim : dimensions) {
- sortHierarchiesByLevel(dim.getHierarchy());
- }
- }
-
- private void sortDimensionsByID() {
- Collections.sort(dimensions, new Comparator<DimensionDesc>() {
- @Override
- public int compare(DimensionDesc d1, DimensionDesc d2) {
- Integer id1 = d1.getId();
- Integer id2 = d2.getId();
- return id1.compareTo(id2);
- }
- });
- }
-
- private void sortMeasuresByID() {
- if (measures == null) {
- measures = Lists.newArrayList();
- }
-
- // Collections.sort(measures, new Comparator<MeasureDesc>() {
- // @Override
- // public int compare(MeasureDesc m1, MeasureDesc m2) {
- // Integer id1 = m1.getId();
- // Integer id2 = m2.getId();
- // return id1.compareTo(id2);
- // }
- // });
- }
-
- private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) {
- if (hierarchies != null) {
- Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() {
- @Override
- public int compare(HierarchyDesc h1, HierarchyDesc h2) {
- Integer level1 = Integer.parseInt(h1.getLevel());
- Integer level2 = Integer.parseInt(h2.getLevel());
- return level1.compareTo(level2);
- }
- });
- }
- }
-
- public boolean hasMemoryHungryMeasures() {
- for (MeasureDesc measure : measures) {
- if (measure.getFunction().getMeasureType().isMemoryHungry()) {
- return true;
- }
- }
- return false;
- }
-
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
-
- public long[] getAutoMergeTimeRanges() {
- return autoMergeTimeRanges;
- }
-
- public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
- this.autoMergeTimeRanges = autoMergeTimeRanges;
- }
-
- /**
- * Add error info and thrown exception out
- *
- * @param message
- */
- public void addError(String message) {
- addError(message, false);
- }
-
- /**
- * @param message error message
- * @param silent if throw exception
- */
- public void addError(String message, boolean silent) {
- if (!silent) {
- throw new IllegalStateException(message);
- } else {
- this.errors.add(message);
- }
- }
-
- public List<String> getError() {
- return this.errors;
- }
-
- public HBaseMappingDesc getHbaseMapping() {
- return hbaseMapping;
- }
-
- public void setHbaseMapping(HBaseMappingDesc hbaseMapping) {
- this.hbaseMapping = hbaseMapping;
- }
-
- public void setNullStrings(String[] nullStrings) {
- this.nullStrings = nullStrings;
- }
-
- public int getStorageType() {
- return storageType;
- }
-
- void setStorageType(int storageType) {
- this.storageType = storageType;
- }
-
- public int getEngineType() {
- return engineType;
- }
-
- void setEngineType(int engineType) {
- this.engineType = engineType;
- }
-
- public List<TblColRef> getAllColumnsNeedDictionary() {
- List<TblColRef> result = Lists.newArrayList();
-
- for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) {
- TblColRef colRef = rowKeyColDesc.getColRef();
- if (rowkey.isUseDictionary(colRef)) {
- result.add(colRef);
- }
- }
-
- for (MeasureDesc measure : measures) {
- MeasureType<?> aggrType = measure.getFunction().getMeasureType();
- result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/DimensionDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/DimensionDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/DimensionDesc.java
deleted file mode 100644
index 40572cb..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/DimensionDesc.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class DimensionDesc {
-
- @JsonProperty("id")
- private int id;
-
- @JsonProperty("name")
- private String name;
-
- @JsonProperty("hierarchy")
- private boolean isHierarchy;
- @JsonProperty("table")
- private String table;
- @JsonProperty("column")
- private String[] column;
- @JsonProperty("derived")
- private String[] derived;
-
- private TableDesc tableDesc;
- private JoinDesc join;
- private HierarchyDesc[] hierarchy;
-
- // computed
- private TblColRef[] columnRefs;
- private TblColRef[] derivedColRefs;
-
- public void init(CubeDesc cubeDesc, Map<String, TableDesc> tables) {
- if (name != null)
- name = name.toUpperCase();
-
- if (table != null)
- table = table.toUpperCase();
-
- tableDesc = tables.get(this.getTable());
- if (tableDesc == null)
- throw new IllegalStateException("Can't find table " + table + " for dimension " + name);
-
- join = null;
- for (LookupDesc lookup : cubeDesc.getModel().getLookups()) {
- if (lookup.getTable().equalsIgnoreCase(this.getTable())) {
- join = lookup.getJoin();
- break;
- }
- }
-
- if (isHierarchy && this.column.length > 0) {
- List<HierarchyDesc> hierarchyList = new ArrayList<HierarchyDesc>(3);
- for (int i = 0, n = this.column.length; i < n; i++) {
- String aColumn = this.column[i];
- HierarchyDesc aHierarchy = new HierarchyDesc();
- aHierarchy.setLevel(String.valueOf(i + 1));
- aHierarchy.setColumn(aColumn);
- hierarchyList.add(aHierarchy);
- }
-
- this.hierarchy = hierarchyList.toArray(new HierarchyDesc[hierarchyList.size()]);
- }
-
- if (hierarchy != null && hierarchy.length == 0)
- hierarchy = null;
- if (derived != null && derived.length == 0)
- derived = null;
-
- if (hierarchy != null) {
- for (HierarchyDesc h : hierarchy)
- h.setColumn(h.getColumn().toUpperCase());
- }
-
- if (derived != null) {
- StringUtil.toUpperCaseArray(derived, derived);
- }
-
- if (derived != null && join == null) {
- throw new IllegalStateException("Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
- }
- }
-
- public boolean isHierarchyColumn(TblColRef col) {
- if (hierarchy == null)
- return false;
-
- for (HierarchyDesc hier : hierarchy) {
- if (hier.getColumnRef().equals(col))
- return true;
- }
- return false;
- }
-
- public boolean isDerived() {
- return derived != null;
- }
-
- public boolean isHierarchy() {
- return isHierarchy;
- }
-
- public void setHierarchy(boolean isHierarchy) {
- this.isHierarchy = isHierarchy;
- }
-
- public String getTable() {
- return table;
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public int getId() {
- return id;
- }
-
- public void setId(int id) {
- this.id = id;
- }
-
- public JoinDesc getJoin() {
- return join;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public TblColRef[] getColumnRefs() {
- return this.columnRefs;
- }
-
- public void setColumnRefs(TblColRef[] colRefs) {
- this.columnRefs = colRefs;
- }
-
- public String[] getColumn() {
- return this.column;
- }
-
- public void setColumn(String[] column) {
- this.column = column;
- }
-
- public HierarchyDesc[] getHierarchy() {
- return hierarchy;
- }
-
- public void setHierarchy(HierarchyDesc[] hierarchy) {
- this.hierarchy = hierarchy;
- }
-
- public String[] getDerived() {
- return derived;
- }
-
- public void setDerived(String[] derived) {
- this.derived = derived;
- }
-
- public TblColRef[] getDerivedColRefs() {
- return derivedColRefs;
- }
-
- public void setDerivedColRefs(TblColRef[] derivedColRefs) {
- this.derivedColRefs = derivedColRefs;
- }
-
- public TableDesc getTableDesc() {
- return this.tableDesc;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- DimensionDesc that = (DimensionDesc) o;
-
- if (id != that.id)
- return false;
- if (!name.equals(that.name))
- return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = id;
- result = 31 * result + name.hashCode();
- return result;
- }
-
- @Override
- public String toString() {
- return "DimensionDesc [name=" + name + ", join=" + join + ", hierarchy=" + Arrays.toString(hierarchy) + ", table=" + table + ", column=" + Arrays.toString(column) + ", derived=" + Arrays.toString(derived) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnDesc.java
deleted file mode 100644
index d5c0e8b..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnDesc.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.util.Arrays;
-
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnDesc {
-
- @JsonProperty("qualifier")
- private String qualifier;
- @JsonProperty("measure_refs")
- private String[] measureRefs;
-
- // these two will be assembled at runtime
- private MeasureDesc[] measures;
- private int[] measureIndex; // the index on CubeDesc.getMeasures()
- private String columnFamilyName;
-
- public String getQualifier() {
- return qualifier;
- }
-
- public void setQualifier(String qualifier) {
- this.qualifier = qualifier;
- }
-
- public String[] getMeasureRefs() {
- return measureRefs;
- }
-
- public void setMeasureRefs(String[] measureRefs) {
- this.measureRefs = measureRefs;
- }
-
- public int[] getMeasureIndex() {
- return measureIndex;
- }
-
- public void setMeasureIndex(int[] index) {
- this.measureIndex = index;
- }
-
- public MeasureDesc[] getMeasures() {
- return measures;
- }
-
- public void setMeasures(MeasureDesc[] measures) {
- this.measures = measures;
- }
-
- public String getColumnFamilyName() {
- return columnFamilyName;
- }
-
- public void setColumnFamilyName(String columnFamilyName) {
- this.columnFamilyName = columnFamilyName;
- }
-
- public int findMeasure(FunctionDesc function) {
- for (int i = 0; i < measures.length; i++) {
- if (measures[i].getFunction().equals(function)) {
- return i;
- }
- }
- return -1;
- }
-
- public boolean containsMeasure(String refName) {
- for (String ref : measureRefs) {
- if (ref.equals(refName))
- return true;
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((columnFamilyName == null) ? 0 : columnFamilyName.hashCode());
- result = prime * result + ((qualifier == null) ? 0 : qualifier.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- HBaseColumnDesc other = (HBaseColumnDesc) obj;
- if (columnFamilyName == null) {
- if (other.columnFamilyName != null)
- return false;
- } else if (!columnFamilyName.equals(other.columnFamilyName))
- return false;
- if (qualifier == null) {
- if (other.qualifier != null)
- return false;
- } else if (!qualifier.equals(other.qualifier))
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return "HBaseColumnDesc [qualifier=" + qualifier + ", measureRefs=" + Arrays.toString(measureRefs) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnFamilyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnFamilyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnFamilyDesc.java
deleted file mode 100644
index 86ba875..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseColumnFamilyDesc.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.util.Arrays;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseColumnFamilyDesc {
-
- @JsonProperty("name")
- private String name;
- @JsonProperty("columns")
- private HBaseColumnDesc[] columns;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public HBaseColumnDesc[] getColumns() {
- return columns;
- }
-
- public void setColumns(HBaseColumnDesc[] columns) {
- this.columns = columns;
- }
-
- @Override
- public String toString() {
- return "HBaseColumnFamilyDesc [name=" + name + ", columns=" + Arrays.toString(columns) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseMappingDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseMappingDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseMappingDesc.java
deleted file mode 100644
index 480dd03..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HBaseMappingDesc.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HBaseMappingDesc {
-
- @JsonProperty("column_family")
- private HBaseColumnFamilyDesc[] columnFamily;
-
- // point to the cube instance which contain this HBaseMappingDesc instance.
- private CubeDesc cubeRef;
-
- public Collection<HBaseColumnDesc> findHBaseColumnByFunction(FunctionDesc function) {
- Collection<HBaseColumnDesc> result = new LinkedList<HBaseColumnDesc>();
- HBaseMappingDesc hbaseMapping = cubeRef.getHBaseMapping();
- if (hbaseMapping == null || hbaseMapping.getColumnFamily() == null) {
- return result;
- }
- for (HBaseColumnFamilyDesc cf : hbaseMapping.getColumnFamily()) {
- for (HBaseColumnDesc c : cf.getColumns()) {
- for (MeasureDesc m : c.getMeasures()) {
- if (m.getFunction().equals(function)) {
- result.add(c);
- }
- }
- }
- }
- return result;
- }
-
- public CubeDesc getCubeRef() {
- return cubeRef;
- }
-
- public void setCubeRef(CubeDesc cubeRef) {
- this.cubeRef = cubeRef;
- }
-
- public HBaseColumnFamilyDesc[] getColumnFamily() {
- return columnFamily;
- }
-
- public void setColumnFamily(HBaseColumnFamilyDesc[] columnFamily) {
- this.columnFamily = columnFamily;
- }
-
- public void init(CubeDesc cubeDesc) {
- cubeRef = cubeDesc;
-
- for (HBaseColumnFamilyDesc cf : columnFamily) {
- cf.setName(cf.getName().toUpperCase());
-
- for (HBaseColumnDesc c : cf.getColumns()) {
- c.setQualifier(c.getQualifier().toUpperCase());
- StringUtil.toUpperCaseArray(c.getMeasureRefs(), c.getMeasureRefs());
- }
- }
- }
-
- @Override
- public String toString() {
- return "HBaseMappingDesc [columnFamily=" + Arrays.toString(columnFamily) + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HierarchyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HierarchyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HierarchyDesc.java
deleted file mode 100644
index a8180a4..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/HierarchyDesc.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class HierarchyDesc {
-
- @JsonProperty("level")
- private String level;
- @JsonProperty("column")
- private String column;
-
- private TblColRef columnRef;
-
- public String getLevel() {
- return level;
- }
-
- public void setLevel(String level) {
- this.level = level;
- }
-
- public TblColRef getColumnRef() {
- return columnRef;
- }
-
- public void setColumnRef(TblColRef column) {
- this.columnRef = column;
- }
-
- public String getColumn() {
- return column;
- }
-
- public void setColumn(String columnName) {
- this.column = columnName;
- }
-
- @Override
- public String toString() {
- return "HierarchyDesc [level=" + level + ", column=" + column + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyColDesc.java
deleted file mode 100644
index eb1b271..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyColDesc.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author yangli9
- *
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyColDesc {
-
- @JsonProperty("column")
- private String column;
- @JsonProperty("length")
- private int length;
- @JsonProperty("dictionary")
- private String dictionary;
- @JsonProperty("mandatory")
- private boolean mandatory = false;
-
- // computed
- private int bitIndex;
- private TblColRef colRef;
-
- public String getDictionary() {
- return dictionary;
- }
-
- public String getColumn() {
- return column;
- }
-
- void setColumn(String column) {
- this.column = column;
- }
-
- public int getLength() {
- return length;
- }
-
- public boolean isMandatory() {
- return mandatory;
- }
-
- public int getBitIndex() {
- return bitIndex;
- }
-
- void setBitIndex(int index) {
- this.bitIndex = index;
- }
-
- public TblColRef getColRef() {
- return colRef;
- }
-
- void setColRef(TblColRef colRef) {
- this.colRef = colRef;
- }
-
- public void setDictionary(String dictionary) {
- this.dictionary = dictionary;
- }
-
- @Override
- public String toString() {
- return "RowKeyColDesc [column=" + column + ", length=" + length + ", dictionary=" + dictionary + ", mandatory=" + mandatory + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyDesc.java
deleted file mode 100644
index dee5543..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/v3/RowKeyDesc.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.model.v3;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyDesc {
-
- public static class HierarchyMask {
- public long fullMask;
- public long[] allMasks;
- }
-
- public static class AggrGroupMask {
- public AggrGroupMask(int size) {
- groupOneBitMasks = new long[size];
- }
-
- public long groupMask;
- public long groupOneBitMasks[];
- public long uniqueMask;
- public long leftoverMask;
- }
-
- @JsonProperty("rowkey_columns")
- private RowKeyColDesc[] rowkeyColumns;
- @JsonProperty("aggregation_groups")
- private String[][] aggregationGroups;
-
- // computed content
- private CubeDesc cubeDesc;
- private Map<TblColRef, RowKeyColDesc> columnMap;
-
- private long fullMask;
- private long mandatoryColumnMask;
- private AggrGroupMask[] aggrGroupMasks;
- private long aggrGroupFullMask;
- private long tailMask;
-
- private List<HierarchyMask> hierarchyMasks;
-
- public RowKeyColDesc[] getRowKeyColumns() {
- return rowkeyColumns;
- }
-
- // search a specific row key col
- public int getRowKeyIndexByColumnName(String columnName) {
- if (this.rowkeyColumns == null)
- return -1;
-
- for (int i = 0; i < this.rowkeyColumns.length; ++i) {
- RowKeyColDesc desc = this.rowkeyColumns[i];
- if (desc.getColumn().equalsIgnoreCase(columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- public int getNCuboidBuildLevels() {
- // N aggregation columns requires N levels of cuboid build
- // - N columns requires N-1 levels build
- // - zero tail cuboid needs one more additional level
- Set<String> aggDims = new HashSet<String>();
- for (String[] aggrGroup : aggregationGroups) {
- for (String dim : aggrGroup) {
- aggDims.add(dim);
- }
- }
- return aggDims.size();
- }
-
- public String[][] getAggregationGroups() {
- return aggregationGroups;
- }
-
- public CubeDesc getCubeRef() {
- return cubeDesc;
- }
-
- public void setCubeRef(CubeDesc cubeRef) {
- this.cubeDesc = cubeRef;
- }
-
- public long getFullMask() {
- return fullMask;
- }
-
- public long getMandatoryColumnMask() {
- return mandatoryColumnMask;
- }
-
- public long getAggrGroupFullMask() {
- return aggrGroupFullMask;
- }
-
- public AggrGroupMask[] getAggrGroupMasks() {
- return aggrGroupMasks;
- }
-
- public List<HierarchyMask> getHierarchyMasks() {
- return hierarchyMasks;
- }
-
- public long getTailMask() {
- return tailMask;
- }
-
- public int getColumnBitIndex(TblColRef col) {
- return getColDesc(col).getBitIndex();
- }
-
- public int getColumnLength(TblColRef col) {
- return getColDesc(col).getLength();
- }
-
- public String getDictionary(TblColRef col) {
- return getColDesc(col).getDictionary();
- }
-
- private RowKeyColDesc getColDesc(TblColRef col) {
- RowKeyColDesc desc = columnMap.get(col);
- if (desc == null)
- throw new NullPointerException("Column " + col + " does not exist in row key desc");
- return desc;
- }
-
- public boolean isUseDictionary(int index) {
- String useDictionary = rowkeyColumns[index].getDictionary();
- return useDictionary(useDictionary);
- }
-
- public boolean isUseDictionary(TblColRef col) {
- String useDictionary = getDictionary(col);
- return useDictionary(useDictionary);
- }
-
- private boolean useDictionary(String useDictionary) {
- return !StringUtils.isBlank(useDictionary) && !"false".equals(useDictionary);
- }
-
- public void init(CubeDesc cube) {
- setCubeRef(cube);
- Map<String, TblColRef> colNameAbbr = cube.buildColumnNameAbbreviation();
-
- buildRowKey(colNameAbbr);
- buildAggregationGroups(colNameAbbr);
- buildHierarchyMasks();
- }
-
- @Override
- public String toString() {
- return "RowKeyDesc [rowkeyColumns=" + Arrays.toString(rowkeyColumns) + ", aggregationGroups=" + Arrays.toString(aggregationGroups) + "]";
- }
-
- private void buildRowKey(Map<String, TblColRef> colNameAbbr) {
- columnMap = new HashMap<TblColRef, RowKeyColDesc>();
- mandatoryColumnMask = 0;
-
- for (int i = 0; i < rowkeyColumns.length; i++) {
- RowKeyColDesc rowKeyColDesc = rowkeyColumns[i];
- String column = rowKeyColDesc.getColumn();
- rowKeyColDesc.setColumn(column.toUpperCase());
- rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1);
- rowKeyColDesc.setColRef(colNameAbbr.get(column));
- if (rowKeyColDesc.getColRef() == null) {
- throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc);
- }
-
- columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc);
-
- if (rowKeyColDesc.isMandatory()) {
- mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex();
- }
- }
- }
-
- private void buildAggregationGroups(Map<String, TblColRef> colNameAbbr) {
- if (aggregationGroups == null) {
- aggregationGroups = new String[0][];
- }
-
- for (int i = 0; i < aggregationGroups.length; i++) {
- StringUtil.toUpperCaseArray(aggregationGroups[i], this.aggregationGroups[i]);
- }
-
- for (int i = 0; i < this.rowkeyColumns.length; i++) {
- int index = rowkeyColumns[i].getBitIndex();
- this.fullMask |= 1L << index;
- }
-
- this.aggrGroupMasks = new AggrGroupMask[aggregationGroups.length];
- for (int i = 0; i < this.aggregationGroups.length; i++) {
- String[] aggGrp = this.aggregationGroups[i];
- AggrGroupMask mask = new AggrGroupMask(aggGrp.length);
-
- for (int j = 0; j < aggGrp.length; j++) {
- TblColRef aggCol = colNameAbbr.get(aggGrp[j].toUpperCase());
- if (aggCol == null) {
- throw new IllegalArgumentException("Can't find aggregation column " + aggGrp[j] + " in cube " + this.cubeDesc.getName());
- }
- Integer index = getColumnBitIndex(aggCol);
- mask.groupMask |= 1L << index;
- mask.groupOneBitMasks[j] = 1L << index;
- this.aggrGroupFullMask |= 1L << index;
- }
- this.aggrGroupMasks[i] = mask;
- }
-
- this.tailMask = fullMask ^ mandatoryColumnMask ^ aggrGroupFullMask;
-
- // unique mask = (bits in this group) - (bits in following groups)
- // leftover mask = (tail bits) + (bits in following groups) - (bits in
- // this group)
- for (int i = 0; i < aggrGroupMasks.length; i++) {
- AggrGroupMask mask = aggrGroupMasks[i];
-
- mask.uniqueMask = mask.groupMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.uniqueMask &= ~aggrGroupMasks[j].groupMask;
- }
-
- mask.leftoverMask = tailMask;
- for (int j = i + 1; j < aggrGroupMasks.length; j++) {
- mask.leftoverMask |= aggrGroupMasks[j].groupMask;
- }
- mask.leftoverMask &= ~mask.groupMask;
- }
- }
-
- private void buildHierarchyMasks() {
- this.hierarchyMasks = new ArrayList<HierarchyMask>();
-
- for (DimensionDesc dimension : this.cubeDesc.getDimensions()) {
- HierarchyDesc[] hierarchies = dimension.getHierarchy();
- if (hierarchies == null || hierarchies.length == 0)
- continue;
-
- HierarchyMask mask = new HierarchyMask();
- ArrayList<Long> allMaskList = new ArrayList<Long>();
- for (int i = 0; i < hierarchies.length; i++) {
- TblColRef hColumn = hierarchies[i].getColumnRef();
- Integer index = getColumnBitIndex(hColumn);
- long bit = 1L << index;
-
- if ((tailMask & bit) > 0)
- continue; // ignore levels in tail, they don't participate
- // aggregation group combination anyway
-
- mask.fullMask |= bit;
- allMaskList.add(mask.fullMask);
- }
-
- mask.allMasks = new long[allMaskList.size()];
- for (int i = 0; i < allMaskList.size(); i++)
- mask.allMasks[i] = allMaskList.get(i);
-
- this.hierarchyMasks.add(mask);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
deleted file mode 100644
index ce45c00..0000000
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.upgrade;
-
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.project.ProjectManager;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by dongli on 11/17/15.
- */
-public class CubeDescSignatureUpdate {
- private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
- private KylinConfig config = null;
- private ResourceStore store;
- private String[] cubeNames;
- private List<String> updatedResources = Lists.newArrayList();
- private List<String> errorMsgs = Lists.newArrayList();
-
- public CubeDescSignatureUpdate(String[] cubes) {
- config = KylinConfig.getInstanceFromEnv();
- store = ResourceStore.getStore(config);
- cubeNames = cubes;
- }
-
- public static void main(String args[]) {
- if (args != null && args.length > 1) {
- System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 ");
- return;
- }
-
- CubeDescSignatureUpdate metadataUpgrade = new CubeDescSignatureUpdate(args);
- metadataUpgrade.update();
-
- logger.info("=================================================================");
- logger.info("Run CubeDescSignatureUpdate completed;");
-
- if (!metadataUpgrade.updatedResources.isEmpty()) {
- logger.info("Following resources are updated successfully:");
- for (String s : metadataUpgrade.updatedResources) {
- logger.info(s);
- }
- } else {
- logger.warn("No resource updated.");
- }
-
- if (!metadataUpgrade.errorMsgs.isEmpty()) {
- logger.info("Here are the error/warning messages, you may need to check:");
- for (String s : metadataUpgrade.errorMsgs) {
- logger.warn(s);
- }
- } else {
- logger.info("No error or warning messages; The update succeeds.");
- }
-
- logger.info("=================================================================");
- }
-
- public void update() {
- logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
- CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
- List<CubeDesc> cubeDescs;
- if (ArrayUtils.isEmpty(cubeNames)) {
- cubeDescs = cubeDescManager.listAllDesc();
- } else {
- String[] names = cubeNames[0].split(",");
- if (ArrayUtils.isEmpty(names))
- return;
- cubeDescs = Lists.newArrayListWithCapacity(names.length);
- for (String name : names) {
- cubeDescs.add(cubeDescManager.getCubeDesc(name));
- }
- }
- for (CubeDesc cubeDesc : cubeDescs) {
- updateCubeDesc(cubeDesc);
- }
-
- verify();
- }
-
- private void verify() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
- CubeDescManager.getInstance(config);
- CubeManager.getInstance(config);
- ProjectManager.getInstance(config);
- }
-
- private void updateCubeDesc(CubeDesc cubeDesc) {
- try {
- cubeDesc.setPartitionDateStart(cubeDesc.getModel().getPartitionDesc().getPartitionDateStart());
- String calculatedSign = cubeDesc.calculateSignature();
- if (!cubeDesc.getSignature().equals(calculatedSign)) {
- cubeDesc.setSignature(calculatedSign);
- store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
- updatedResources.add(cubeDesc.getResourcePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2512fb6f/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java
new file mode 100644
index 0000000..3892a48
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/V1_5_0/CubeDescUpgrade_v_1_5_0.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.V1_5_0;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.model.AggregationGroup;
+import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.cube.model.v1_4_0.CubeDesc;
+import org.apache.kylin.cube.model.v1_4_0.DimensionDesc;
+import org.apache.kylin.cube.model.v1_4_0.HBaseMappingDesc;
+import org.apache.kylin.cube.model.v1_4_0.RowKeyColDesc;
+import org.apache.kylin.cube.model.v1_4_0.RowKeyDesc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class CubeDescUpgrade_v_1_5_0 {
+
+ @SuppressWarnings("unused")
+ private static final Logger logger = LoggerFactory.getLogger(CubeDescUpgrade_v_1_5_0.class);
+
+ private static final Serializer<CubeDesc> oldCubeDescSerializer = new JsonSerializer<>(CubeDesc.class);
+
+ private ResourceStore store;
+ private String resourcePath;
+
+ private List<String[]> oldHierarchies = Lists.newArrayList();
+ private List<String> oldMandatories = Lists.newArrayList();
+ private String[][] oldAggGroup = null;
+ private Set<String> allRowKeyCols = newIgnoreCaseSet(null);
+
+ public CubeDescUpgrade_v_1_5_0(String resourcePath, ResourceStore resourceStore) {
+ this.resourcePath = resourcePath;
+ this.store = resourceStore;
+ }
+
+ public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
+ CubeDesc oldModel = loadOldCubeDesc(resourcePath);
+
+ org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
+ copyUnChangedProperties(oldModel, newModel);
+ upgradeDimension(oldModel, newModel);
+ upgradeRowKey(oldModel, newModel);
+ upgradeHBaseMapping(oldModel, newModel);
+ upgradeAggregationGroup(newModel);//must do at last
+
+ return newModel;
+ }
+
+ private CubeDesc loadOldCubeDesc(String path) throws IOException {
+ CubeDesc ndesc = store.getResource(path, CubeDesc.class, oldCubeDescSerializer);
+
+ if (StringUtils.isBlank(ndesc.getName())) {
+ throw new IllegalStateException("CubeDesc name must not be blank");
+ }
+
+ return ndesc;
+ }
+
+ private Set<String> newIgnoreCaseSet(Set<String> input) {
+ Set<String> ret = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ if (input != null)
+ ret.addAll(input);
+ return ret;
+ }
+
+ private String[] toArray(Set<String> input) {
+ return input.toArray(new String[input.size()]);
+ }
+
+ private boolean rowKeyColExistsInMultipleAggGroup() {
+ if (oldAggGroup == null)
+ return false;
+
+ int total = 0;
+ Set<String> overall = newIgnoreCaseSet(null);
+ for (String[] group : oldAggGroup) {
+ Set<String> temp = newIgnoreCaseSet(null);
+ for (String entry : group) {
+
+ overall.add(entry);
+ temp.add(entry);
+ }
+ total += temp.size();
+ }
+ return overall.size() != total;
+ }
+
+ private void upgradeAggregationGroup(org.apache.kylin.cube.model.CubeDesc newModel) {
+
+ List<AggregationGroup> aggs = Lists.newArrayList();
+ if (oldAggGroup == null || oldAggGroup.length == 0) {
+ oldAggGroup = new String[1][];
+ oldAggGroup[0] = toArray(allRowKeyCols);
+ }
+
+ if (rowKeyColExistsInMultipleAggGroup()) {
+ throw new IllegalArgumentException("rowKeyColExistsInMultipleAggGroup!");
+ }
+
+ Set<String> visited = newIgnoreCaseSet(null);
+
+ for (String[] group : oldAggGroup) {
+ AggregationGroup agg = new AggregationGroup();
+
+ Set<String> remaining = newIgnoreCaseSet(allRowKeyCols);
+ remaining.removeAll(visited);
+
+ Set<String> joint = newIgnoreCaseSet(remaining);
+ joint.removeAll(oldMandatories);
+
+ Set<String> groupAsSet = newIgnoreCaseSet(null);
+ for (String entry : group) {
+ groupAsSet.add(entry);
+ }
+ visited.addAll(groupAsSet);
+ joint.removeAll(groupAsSet);
+
+ List<String> mandatories = Lists.newArrayList();
+ List<String[]> hierarchies = Lists.newArrayList();
+
+ for (String s : oldMandatories) {
+ mandatories.add(s);
+ }
+
+ for (String[] h : oldHierarchies) {
+ if (groupAsSet.containsAll(Arrays.asList(h))) {
+ hierarchies.add(h);
+ }
+ }
+
+ agg.setIncludes(toArray(remaining));
+
+ SelectRule selectRule = new SelectRule();
+ selectRule.hierarchy_dims = hierarchies.toArray(new String[hierarchies.size()][]);
+ if (joint.size() != 0) {
+ selectRule.joint_dims = new String[1][];
+ selectRule.joint_dims[0] = joint.toArray(new String[joint.size()]);
+ } else {
+ selectRule.joint_dims = new String[0][];
+ }
+ selectRule.mandatory_dims = mandatories.toArray(new String[mandatories.size()]);
+ agg.setSelectRule(selectRule);
+
+ aggs.add(agg);
+
+ }
+ newModel.setAggregationGroups(aggs);
+ }
+
+ private void upgradeDimension(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ List<DimensionDesc> oldDimensions = oldModel.getDimensions();
+ if (oldDimensions == null) {
+ throw new IllegalArgumentException("dimensions is null");
+ }
+ List<org.apache.kylin.cube.model.DimensionDesc> newDimensions = Lists.newArrayList();
+
+ for (DimensionDesc oldDim : oldDimensions) {
+ if (oldDim.isDerived()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn("{FK}");
+ newDim.setDerived(oldDim.getDerived());
+
+ newDimensions.add(newDim);
+ } else {
+ if (oldDim.isHierarchy()) {
+ oldHierarchies.add(oldDim.getColumn());
+ }
+
+ for (String columnStr : oldDim.getColumn()) {
+ org.apache.kylin.cube.model.DimensionDesc newDim = new org.apache.kylin.cube.model.DimensionDesc();
+
+ newDim.setName(oldDim.getName());
+ newDim.setTable(oldDim.getTable());
+ newDim.setColumn(columnStr);
+ newDim.setDerived(null);
+
+ newDimensions.add(newDim);
+ }
+ }
+ }
+
+ newModel.setDimensions(newDimensions);
+ }
+
+ private void upgradeRowKey(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ RowKeyDesc oldRowKey = oldModel.getRowkey();
+ if (oldModel == null) {
+ throw new IllegalArgumentException("RowKeyDesc is null");
+ }
+
+ if (oldRowKey.getRowKeyColumns() == null) {
+ throw new IllegalArgumentException("RowKeyDesc.getRowKeyColumns is null");
+ }
+
+ org.apache.kylin.cube.model.RowKeyDesc newRowKey = new org.apache.kylin.cube.model.RowKeyDesc();
+ org.apache.kylin.cube.model.RowKeyColDesc[] cols = new org.apache.kylin.cube.model.RowKeyColDesc[oldRowKey.getRowKeyColumns().length];
+ int index = 0;
+ for (RowKeyColDesc oldRowKeyCol : oldRowKey.getRowKeyColumns()) {
+ org.apache.kylin.cube.model.RowKeyColDesc newRowKeyCol = new org.apache.kylin.cube.model.RowKeyColDesc();
+
+ allRowKeyCols.add(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.isMandatory()) {
+ oldMandatories.add(oldRowKeyCol.getColumn());
+ }
+
+ newRowKeyCol.setColumn(oldRowKeyCol.getColumn());
+ if (oldRowKeyCol.getDictionary() != null && "true".equalsIgnoreCase(oldRowKeyCol.getDictionary())) {
+ newRowKeyCol.setEncoding("dict");
+ } else if (oldRowKeyCol.getLength() > 0) {
+ newRowKeyCol.setEncoding("fixed_length:" + oldRowKeyCol.getLength());
+ } else {
+ throw new IllegalArgumentException("Unknow encoding: Dictionary " + oldRowKeyCol.getDictionary() + ", length: " + oldRowKeyCol.getLength());
+ }
+ cols[index++] = newRowKeyCol;
+ }
+ oldAggGroup = oldRowKey.getAggregationGroups();
+
+ newRowKey.setRowkeyColumns(cols);
+ newModel.setRowkey(newRowKey);
+ }
+
+ private void upgradeHBaseMapping(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ HBaseMappingDesc hbaseMappingDesc = oldModel.getHBaseMapping();
+ try {
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, hbaseMappingDesc);
+ byte[] blob = os.toByteArray();
+ ByteArrayInputStream is = new ByteArrayInputStream(blob);
+ org.apache.kylin.cube.model.HBaseMappingDesc newHBaseMappingDesc = JsonUtil.readValue(is, org.apache.kylin.cube.model.HBaseMappingDesc.class);
+ newModel.setHbaseMapping(newHBaseMappingDesc);
+
+ } catch (IOException e) {
+ throw new RuntimeException("error when copying HBaseMappingDesc");
+ }
+ }
+
+ private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
+ newModel.setUuid(oldModel.getUuid());
+ newModel.setName(oldModel.getName());
+ newModel.setDescription(oldModel.getDescription());
+ newModel.setMeasures(oldModel.getMeasures());
+ newModel.setNullStrings(oldModel.getNullStrings());
+ newModel.setModelName(oldModel.getModelName());
+ newModel.setNotifyList(oldModel.getNotifyList());
+ newModel.setLastModified(oldModel.getLastModified());
+ newModel.setStorageType(oldModel.getStorageType());
+ newModel.setEngineType(oldModel.getEngineType());
+ }
+
+}