You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/07/22 06:13:06 UTC
[16/47] incubator-kylin git commit: KYLIN-875 rename modules:
core-common, core-cube, core-dictionary, core-cube
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
deleted file mode 100644
index d406da1..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.MetadataManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Manager class for CubeDesc; extracted from #CubeManager
- * @author shaoshi
- *
- */
-public class CubeDescManager {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeDescManager.class);
-
- public static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
-
- // ============================================================================
-
- private KylinConfig config;
- // name ==> CubeDesc
- private CaseInsensitiveStringCache<CubeDesc> cubeDescMap = new CaseInsensitiveStringCache<CubeDesc>(Broadcaster.TYPE.CUBE_DESC);
-
- public static CubeDescManager getInstance(KylinConfig config) {
- CubeDescManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeDescManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeDescManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeDescManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- private CubeDescManager(KylinConfig config) throws IOException {
- logger.info("Initializing CubeDescManager with config " + config);
- this.config = config;
- reloadAllCubeDesc();
- }
-
- public CubeDesc getCubeDesc(String name) {
- return cubeDescMap.get(name);
- }
-
- public List<CubeDesc> listAllDesc(){
- return new ArrayList<CubeDesc>(cubeDescMap.values());
- }
-
- /**
- * Reload CubeDesc from resource store It will be triggered by an desc
- * update event.
- *
- * @param name
- * @throws IOException
- */
- public CubeDesc reloadCubeDescLocal(String name) throws IOException {
-
- // Save Source
- String path = CubeDesc.getCubeDescResourcePath(name);
-
- // Reload the CubeDesc
- CubeDesc ndesc = loadCubeDesc(path);
-
- // Here replace the old one
- cubeDescMap.putLocal(ndesc.getName(), ndesc);
- return ndesc;
- }
-
- private CubeDesc loadCubeDesc(String path) throws IOException {
- ResourceStore store = getStore();
- CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("CubeDesc name must not be blank");
- }
-
- ndesc.init(config, getMetadataManager().getAllTablesMap());
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
- }
-
- return ndesc;
- }
-
- /**
- * Create a new CubeDesc
- *
- * @param cubeDesc
- * @return
- * @throws IOException
- */
- public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException {
- if (cubeDesc.getUuid() == null || cubeDesc.getName() == null)
- throw new IllegalArgumentException();
- if (cubeDescMap.containsKey(cubeDesc.getName()))
- throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists");
-
- try {
- cubeDesc.init(config, getMetadataManager().getAllTablesMap());
- } catch (IllegalStateException e) {
- cubeDesc.addError(e.getMessage(), true);
- }
- // Check base validation
- if (!cubeDesc.getError().isEmpty()) {
- return cubeDesc;
- }
- // Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
- ValidateContext context = validator.validate(cubeDesc, true);
- if (!context.ifPass()) {
- return cubeDesc;
- }
-
- cubeDesc.setSignature(cubeDesc.calculateSignature());
-
- String path = cubeDesc.getResourcePath();
- getStore().putResource(path, cubeDesc, CUBE_DESC_SERIALIZER);
- cubeDescMap.put(cubeDesc.getName(), cubeDesc);
-
- return cubeDesc;
- }
-
- // remove cubeDesc
- public void removeCubeDesc(CubeDesc cubeDesc) throws IOException {
- String path = cubeDesc.getResourcePath();
- getStore().deleteResource(path);
- cubeDescMap.remove(cubeDesc.getName());
- }
-
- // remove cubeDesc
- public void removeLocalCubeDesc(String name) throws IOException {
- cubeDescMap.removeLocal(name);
- }
-
- private void reloadAllCubeDesc() throws IOException {
- ResourceStore store = getStore();
- logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
-
- cubeDescMap.clear();
-
- List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_DESC_RESOURCE_ROOT, MetadataConstants.FILE_SURFIX);
- for (String path : paths) {
- CubeDesc desc;
- try {
- desc = loadCubeDesc(path);
- } catch (Exception e) {
- logger.error("Error loading cube desc " + path, e);
- continue;
- }
- if (path.equals(desc.getResourcePath()) == false) {
- logger.error("Skip suspicious desc at " + path + ", " + desc + " should be at " + desc.getResourcePath());
- continue;
- }
- if (cubeDescMap.containsKey(desc.getName())) {
- logger.error("Dup CubeDesc name '" + desc.getName() + "' on path " + path);
- continue;
- }
-
- cubeDescMap.putLocal(desc.getName(), desc);
- }
-
- logger.debug("Loaded " + cubeDescMap.size() + " Cube(s)");
- }
-
- /**
- * Update CubeDesc with the input. Broadcast the event into cluster
- *
- * @param desc
- * @return
- * @throws IOException
- */
- public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
- // Validate CubeDesc
- if (desc.getUuid() == null || desc.getName() == null) {
- throw new IllegalArgumentException();
- }
- String name = desc.getName();
- if (!cubeDescMap.containsKey(name)) {
- throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist.");
- }
-
- try {
- desc.init(config, getMetadataManager().getAllTablesMap());
- } catch (IllegalStateException e) {
- desc.addError(e.getMessage(), true);
- return desc;
- } catch (IllegalArgumentException e) {
- desc.addError(e.getMessage(), true);
- return desc;
- }
-
- // Semantic validation
- CubeMetadataValidator validator = new CubeMetadataValidator();
- ValidateContext context = validator.validate(desc, true);
- if (!context.ifPass()) {
- return desc;
- }
-
- desc.setSignature(desc.calculateSignature());
-
- // Save Source
- String path = desc.getResourcePath();
- getStore().putResource(path, desc, CUBE_DESC_SERIALIZER);
-
- // Reload the CubeDesc
- CubeDesc ndesc = loadCubeDesc(path);
- // Here replace the old one
- cubeDescMap.put(ndesc.getName(), desc);
-
- return ndesc;
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
deleted file mode 100644
index 139ec2f..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.v1.CubeDesc;
-import org.apache.kylin.cube.model.v1.CubePartitionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.metadata.model.PartitionDesc;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-public class CubeDescUpgrader {
-
- private String resourcePath;
-
- @SuppressWarnings("unused")
- private static final Log logger = LogFactory.getLog(CubeDescUpgrader.class);
-
- private static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER_V1 = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- public CubeDescUpgrader(String resourcePath) {
- this.resourcePath = resourcePath;
- }
-
- public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
- CubeDesc oldModel = loadOldCubeDesc(resourcePath);
-
- org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
-
- copyUnChangedProperties(oldModel, newModel);
-
- DataModelDesc model = extractDataModel(oldModel, newModel);
- newModel.setModel(model);
-
- updateDimensions(oldModel, newModel);
-
- updateRowkeyDictionary(oldModel, newModel);
-
- return newModel;
- }
-
- private void updateRowkeyDictionary(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- RowKeyDesc rowKey = newModel.getRowkey();
-
- for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) {
- if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0)
- rowkeyCol.setDictionary("true");
- }
-
- }
-
- private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- newModel.setUuid(oldModel.getUuid());
- newModel.setName(oldModel.getName());
- newModel.setDescription(oldModel.getDescription());
- newModel.setNullStrings(oldModel.getNullStrings());
-
- newModel.setMeasures(oldModel.getMeasures());
- newModel.setRowkey(oldModel.getRowkey());
- newModel.setHbaseMapping(oldModel.getHBaseMapping());
-
- newModel.setSignature(oldModel.getSignature());
-
- newModel.setNotifyList(oldModel.getNotifyList());
- newModel.setLastModified(oldModel.getLastModified());
- }
-
- private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) {
- DimensionDesc newDim = new DimensionDesc();
-
- newDim.setId(dimId);
- newDim.setName(name);
- newDim.setTable(getMetadataManager().appendDBName(dim.getTable()));
-
- return newDim;
- }
-
- private void updateDimensions(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<DimensionDesc> newDimensions = Lists.newArrayList();
- newModel.setDimensions(newDimensions);
-
- int dimId = 0;
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
-
- DimensionDesc newDim = null;
- // if a dimension defines "column", "derived" and "hierarchy" at the same time, separate it into three dimensions;
-
- boolean needNameSuffix = false;
- if (dim.getColumn() != null && !"{FK}".equals(dim.getColumn())) {
- //column on fact table
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setColumn(new String[]{dim.getColumn()});
- needNameSuffix = true;
- } else if (ArrayUtils.isEmpty(dim.getDerived()) && ArrayUtils.isEmpty(dim.getHierarchy())) {
- // user defines a lookup table, but didn't use any column other than the pk, in this case, convert to use fact table's fk
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setTable(getMetadataManager().appendDBName(newModel.getFactTable()));
-
- newDim.setColumn(dim.getJoin().getForeignKey());
- }
-
- if (!ArrayUtils.isEmpty(dim.getDerived())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_DERIVED" : ""));
- newDimensions.add(newDim);
- newDim.setDerived(dim.getDerived());
- newDim.setColumn(null); // derived column must come from a lookup table; in this case the fk will be the dimension column, no need to explicitly declare it;
- needNameSuffix = true;
- }
-
- if (!ArrayUtils.isEmpty(dim.getHierarchy())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_HIERARCHY" : ""));
- newDimensions.add(newDim);
-
- newDim.setHierarchy(true);
-
- List<String> columns = Lists.newArrayList();
- for (HierarchyDesc hierarch : dim.getHierarchy()) {
- String col = hierarch.getColumn();
- columns.add(col);
- }
-
- newDim.setColumn(columns.toArray(new String[columns.size()]));
- }
-
- }
- }
-
- private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- DataModelDesc dm = new DataModelDesc();
- dm.setUuid(UUID.randomUUID().toString());
- String factTable = oldModel.getFactTable();
- dm.setName(oldModel.getName());
- dm.setFactTable(getMetadataManager().appendDBName(factTable));
-
- newModel.setModelName(dm.getName());
-
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<LookupDesc> lookups = Lists.newArrayList();
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
- JoinDesc join = dim.getJoin();
- if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) {
- LookupDesc lookup = new LookupDesc();
- lookup.setJoin(join);
- String table = dim.getTable();
- lookup.setTable(getMetadataManager().appendDBName(table));
-
- lookups.add(lookup);
- }
- }
-
- dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
- dm.setFilterCondition(oldModel.getFilterCondition());
- updatePartitionDesc(oldModel, dm);
-
-
- if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE);
- }
-
- return dm;
- }
-
- private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) {
-
- CubePartitionDesc partition = oldModel.getCubePartitionDesc();
- PartitionDesc newPartition = new PartitionDesc();
-
- if (partition.getPartitionDateColumn() != null) {
- String partitionCol = partition.getPartitionDateColumn();
-
- String[] tablecolumn = partitionCol.split("\\.");
- if (tablecolumn != null && tablecolumn.length == 2) {
- // pattern is <tablename>.<colname>
- String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]);
- newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]);
- } else {
-
- if (partitionCol.indexOf(".") < 0) {
- // pattern is <colname>
- partitionCol = dm.getFactTable() + "." + partitionCol;
- }
-
- newPartition.setPartitionDateColumn(partitionCol);
- }
- }
-
- // only append is supported
- newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND);
-
- newPartition.setPartitionDateStart(partition.getPartitionDateStart());
-
- dm.setPartitionDesc(newPartition);
- }
-
- private CubeDesc loadOldCubeDesc(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("CubeDesc name must not be blank");
- }
-
- Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap();
- Map<String, TableDesc> newMap = Maps.newHashMap();
- for (Entry<String, TableDesc> entry : tableMap.entrySet()) {
- String t = entry.getKey();
-
- if (t.indexOf(".") > 0) {
- newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue());
-
- }
- }
- ndesc.init(KylinConfig.getInstanceFromEnv(), newMap);
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
- }
-
- return ndesc;
- }
-
- private static MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
-
- protected static ResourceStore getStore() {
- return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
deleted file mode 100644
index f028c33..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-import java.util.Collection;
-import java.util.HashSet;
-
-/**
- *
- * the unified logic for defining a sql's dimension
- */
-public class CubeDimensionDeriver {
-
- public static Collection<TblColRef> getDimensionColumns(Collection<TblColRef> groupByColumns, Collection<TblColRef> filterColumns) {
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
deleted file mode 100644
index 43c261f..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.*;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeInstance extends RootPersistentEntity implements IRealization {
-
- public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
- CubeInstance cubeInstance = new CubeInstance();
-
- cubeInstance.setConfig(cubeDesc.getConfig());
- cubeInstance.setName(cubeName);
- cubeInstance.setDescName(cubeDesc.getName());
- cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
- cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
- cubeInstance.updateRandomUuid();
- cubeInstance.setProjectName(projectName);
-
- return cubeInstance;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private RealizationStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<CubeSegment> segments = new ArrayList<CubeSegment>();
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("auto_merge_time_ranges")
- private long[] autoMergeTimeRanges;
-
- @JsonProperty("retention_range")
- private long retentionRange = 0;
-
- private String projectName;
-
- private static final int COST_WEIGHT_DIMENSION = 1;
- private static final int COST_WEIGHT_MEASURE = 1;
- private static final int COST_WEIGHT_LOOKUP_TABLE = 1;
- private static final int COST_WEIGHT_INNER_JOIN = 2;
-
- public List<CubeSegment> getBuildingSegments() {
- List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
- if (null != segments) {
- for (CubeSegment segment : segments) {
- if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
- buildingSegments.add(segment);
- }
- }
- }
-
- return buildingSegments;
- }
-
- public List<CubeSegment> getMergingSegments(CubeSegment mergedSegment) {
- List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>();
- if (null != this.segments) {
- for (CubeSegment segment : this.segments) {
- if (!mergedSegment.equals(segment) //
- && mergedSegment.getDateRangeStart() <= segment.getDateRangeStart() && mergedSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) {
- mergingSegments.add(segment);
- }
- }
- }
- return mergingSegments;
- }
-
- public CubeDesc getDescriptor() {
- return CubeDescManager.getInstance(config).getCubeDesc(descName);
- }
-
- @Override
- public DataModelDesc getDataModelDesc(){return this.getDescriptor().getModel();}
-
- public boolean isReady() {
- return getStatus() == RealizationStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return getCanonicalName();
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("input_records_count")
- public long getInputRecordCount() {
- long sizeRecordCount = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getInputRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("input_records_size")
- public long getInputRecordSize() {
- long sizeRecordSize = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getCanonicalName() {
- return getType() + "[name=" + name + "]";
- }
-
- @Override
- public String getFactTable() {
- return this.getDescriptor().getFactTable();
- }
-
- @Override
- public List<MeasureDesc> getMeasures() {
- return getDescriptor().getMeasures();
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName.toUpperCase();
- }
-
- public String getOriginDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public void setCost(int cost) {
- this.cost = cost;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(RealizationStatusEnum status) {
- this.status = status;
- }
-
- public CubeSegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public CubeSegment getLatestReadySegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getStatus() != SegmentStatusEnum.READY)
- continue;
- if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
- latest = seg;
- }
- }
- return latest;
- }
-
- public List<CubeSegment> getSegments() {
- return segments;
- }
-
- public List<CubeSegment> getSegments(SegmentStatusEnum status) {
- List<CubeSegment> result = new ArrayList<CubeSegment>();
-
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public List<CubeSegment> getSegment(SegmentStatusEnum status) {
- List<CubeSegment> result = Lists.newArrayList();
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
- return result;
- }
-
- public CubeSegment getSegment(String name, SegmentStatusEnum status) {
- for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<CubeSegment> segments) {
- this.segments = segments;
- }
-
- public CubeSegment getSegmentById(String segmentId) {
- for (CubeSegment segment : segments) {
- if (Objects.equal(segment.getUuid(), segmentId)) {
- return segment;
- }
- }
- return null;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- @Override
- public boolean isCapable(SQLDigest digest) {
- return CubeCapabilityChecker.check(this, digest, true);
- }
-
- @Override
- public int getCost(SQLDigest digest) {
- int calculatedCost = cost;
-
- calculatedCost += getAllDimensions().size() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
-
- for (LookupDesc lookupDesc : this.getDescriptor().getModel().getLookups()) {
- // more tables, more cost
- calculatedCost += COST_WEIGHT_LOOKUP_TABLE;
- if ("inner".equals(lookupDesc.getJoin().getType())) {
- // inner join cost is bigger than left join, as it will filter some records
- calculatedCost += COST_WEIGHT_INNER_JOIN;
- }
- }
- return calculatedCost;
- }
-
- @Override
- public RealizationType getType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return Lists.newArrayList(getDescriptor().listAllColumns());
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
- @Override
- public long getDateRangeStart() {
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeStart() < startTime)
- startTime = seg.getDateRangeStart();
- }
-
- return startTime;
- }
-
- @Override
- public long getDateRangeEnd() {
-
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = 0;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeEnd() > endTime)
- endTime = seg.getDateRangeEnd();
- }
-
- return endTime;
- }
-
- @Override
- public String getModelName() {
- return this.getDescriptor().getModelName();
- }
-
- @Override
- public List<TblColRef> getAllDimensions() {
- return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
- }
-
- public long[] getAutoMergeTimeRanges() {
- return autoMergeTimeRanges;
- }
-
- public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
- this.autoMergeTimeRanges = autoMergeTimeRanges;
- }
-
- public boolean needAutoMerge() {
- if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
- return false;
-
- if (this.getDescriptor().hasHolisticCountDistinctMeasures())
- return false;
-
- return autoMergeTimeRanges != null && autoMergeTimeRanges.length > 0;
- }
-
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
deleted file mode 100644
index cc61dac..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ /dev/null
@@ -1,861 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.IRealizationProvider;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.source.ReadableTable;
-import org.apache.kylin.source.TableSourceFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-
-/**
- * @author yangli9
- */
-public class CubeManager implements IRealizationProvider {
-
- private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
- private static int HBASE_TABLE_LENGTH = 10;
- public static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class);
-
- private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>();
-
- public static CubeManager getInstance(KylinConfig config) {
- CubeManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one cubemanager singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // cube name ==> CubeInstance
- private CaseInsensitiveStringCache<CubeInstance> cubeMap = new CaseInsensitiveStringCache<CubeInstance>(Broadcaster.TYPE.CUBE);
- // "table/column" ==> lookup table
- // private SingleValueCache<String, LookupStringTable> lookupTables = new SingleValueCache<String, LookupStringTable>(Broadcaster.TYPE.METADATA);
-
- // for generation hbase table name of a new segment
- private Multimap<String, String> usedStorageLocation = HashMultimap.create();
-
- private CubeManager(KylinConfig config) throws IOException {
- logger.info("Initializing CubeManager with config " + config);
- this.config = config;
-
- loadAllCubeInstance();
- }
-
- public List<CubeInstance> listAllCubes() {
- return new ArrayList<CubeInstance>(cubeMap.values());
- }
-
- public CubeInstance getCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- return cubeMap.get(cubeName);
- }
-
- /**
- * Get related Cubes by cubedesc name. By default, the desc name will be
- * translated into upper case.
- *
- * @param descName CubeDesc name
- * @return
- */
- public List<CubeInstance> getCubesByDesc(String descName) {
-
- descName = descName.toUpperCase();
- List<CubeInstance> list = listAllCubes();
- List<CubeInstance> result = new ArrayList<CubeInstance>();
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- CubeInstance ci = it.next();
- if (descName.equalsIgnoreCase(ci.getDescName())) {
- result.add(ci);
- }
- }
- return result;
- }
-
- public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- if (!cubeDesc.getRowkey().isUseDictionary(col))
- return null;
-
- DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
-
- if (dictInfo != null) {
- cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
-
- CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
- cubeBuilder.setToUpdateSegs(cubeSeg);
- updateCube(cubeBuilder);
- }
- return dictInfo;
- }
-
- /**
- * return null if no dictionary for given column
- */
- public Dictionary<?> getDictionary(CubeSegment cubeSeg, TblColRef col) {
- DictionaryInfo info = null;
- try {
- DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
- String dictResPath = cubeSeg.getDictResPath(col);
- if (dictResPath == null)
- return null;
-
- info = dictMgr.getDictionaryInfo(dictResPath);
- if (info == null)
- throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid cube state; cube segment" + cubeSeg + ", col " + col);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
- }
-
- return info.getDictionaryObject();
- }
-
- public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
- SnapshotManager snapshotMgr = getSnapshotManager();
-
- TableDesc tableDesc = metaMgr.getTableDesc(lookupTable);
- ReadableTable hiveTable = TableSourceFactory.createReadableTable(tableDesc);
- SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);
-
- cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
- CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
- cubeBuilder.setToUpdateSegs(cubeSeg);
- updateCube(cubeBuilder);
-
- return snapshot;
- }
-
- // sync on update
- public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException {
- logger.info("Dropping cube '" + cubeName + "'");
- // load projects before remove cube from project
-
- // delete cube instance and cube desc
- CubeInstance cube = getCube(cubeName);
-
- if (deleteDesc && cube.getDescriptor() != null) {
- CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
- }
-
- // remove cube and update cache
- getStore().deleteResource(cube.getResourcePath());
- cubeMap.remove(cube.getName());
-
- // delete cube from project
- ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
-
- return cube;
- }
-
- // sync on update
- public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
- logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
-
- // save cube resource
- CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
- cube.setOwner(owner);
-
- updateCube(new CubeUpdate(cube));
- ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
-
- return cube;
- }
-
- public CubeInstance updateCube(CubeUpdate cubeBuilder) throws IOException {
- return updateCube(cubeBuilder, 0);
- }
-
- private boolean validateReadySegments(CubeInstance cube) {
- final List<CubeSegment> readySegments = cube.getSegment(SegmentStatusEnum.READY);
- if (readySegments.size() == 0) {
- return true;
- }
- for (CubeSegment readySegment : readySegments) {
- if (readySegment.getDateRangeEnd() <= readySegment.getDateRangeStart()) {
- logger.warn(String.format("segment:%s has invalid date range:[%d, %d], validation failed", readySegment.getName(), readySegment.getDateRangeStart(), readySegment.getDateRangeEnd()));
- return false;
- }
- }
- Collections.sort(readySegments);
- for (int i = 0, size = readySegments.size(); i < size - 1; i++) {
- CubeSegment lastSegment = readySegments.get(i);
- CubeSegment segment = readySegments.get(i + 1);
- if (lastSegment.getDateRangeEnd() <= segment.getDateRangeStart()) {
- continue;
- } else {
- logger.warn(String.format("segment:%s and %s data range has overlap, validation failed", lastSegment.getName(), segment.getName()));
- return false;
- }
- }
- return true;
- }
-
- private CubeInstance updateCube(CubeUpdate cubeBuilder, int retry) throws IOException {
- if (cubeBuilder == null || cubeBuilder.getCubeInstance() == null)
- throw new IllegalStateException();
-
- CubeInstance cube = cubeBuilder.getCubeInstance();
- logger.info("Updating cube instance '" + cube.getName() + "'");
-
- if (cubeBuilder.getToAddSegs() != null)
- cube.getSegments().addAll(Arrays.asList(cubeBuilder.getToAddSegs()));
-
- List<String> toRemoveResources = Lists.newArrayList();
- if (cubeBuilder.getToRemoveSegs() != null) {
- Iterator<CubeSegment> iterator = cube.getSegments().iterator();
- while (iterator.hasNext()) {
- CubeSegment currentSeg = iterator.next();
- for (CubeSegment toRemoveSeg : cubeBuilder.getToRemoveSegs()) {
- if(currentSeg.getUuid().equals(toRemoveSeg.getUuid())) {
- iterator.remove();
- toRemoveResources.add(toRemoveSeg.getStatisticsResourcePath());
- }
- }
- }
-
- }
-
- if (cubeBuilder.getToUpdateSegs() != null) {
- for (CubeSegment segment : cubeBuilder.getToUpdateSegs()) {
- for (int i = 0; i < cube.getSegments().size(); i++) {
- if (cube.getSegments().get(i).getUuid().equals(segment.getUuid())) {
- cube.getSegments().set(i, segment);
- }
- }
-
- }
- }
-
- Collections.sort(cube.getSegments());
-
- if (!validateReadySegments(cube)) {
- throw new IllegalStateException("Has invalid Ready segments in cube " + cube.getName());
- }
-
- if (cubeBuilder.getStatus() != null) {
- cube.setStatus(cubeBuilder.getStatus());
- }
-
- if (cubeBuilder.getOwner() != null) {
- cube.setOwner(cubeBuilder.getOwner());
- }
-
- if (cubeBuilder.getCost() > 0) {
- cube.setCost(cubeBuilder.getCost());
- }
-
- try {
- getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- } catch (IllegalStateException ise) {
- logger.warn("Write conflict to update cube " + cube.getName() + " at try " + retry + ", will retry...");
- if (retry >= 7) {
- logger.error("Retried 7 times till got error, abandoning...", ise);
- throw ise;
- }
-
- cube = reloadCubeLocal(cube.getName());
- cubeBuilder.setCubeInstance(cube);
- retry++;
- cube = updateCube(cubeBuilder, retry);
- }
-
- if (toRemoveResources.size() > 0) {
- for (String resource : toRemoveResources) {
- try {
- getStore().deleteResource(resource);
- } catch (IOException ioe) {
- logger.error("Failed to delete resource " + toRemoveResources.toString());
- }
- }
- }
-
- cubeMap.put(cube.getName(), cube);
-
- //this is a duplicate call to take care of scenarios where REST cache service unavailable
- ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).clearL2Cache();
-
- return cube;
- }
-
- public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- if (cube.getSegments().size() == 0)
- throw new IllegalStateException("expect at least one existing segment");
-
- long appendStart = calculateStartDateForAppendSegment(cube);
- CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
-
- long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
-
- validateNewSegments(cube, mergeSegment);
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube).setToAddSegs(appendSegment, mergeSegment);
- updateCube(cubeBuilder);
-
- return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
- }
-
- public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException {
- return appendSegments(cube, endDate, true, true);
- }
-
- public CubeSegment appendSegments(CubeInstance cube, long endDate, boolean strictChecking, boolean saveChange) throws IOException {
- long startDate = 0;
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
- startDate = calculateStartDateForAppendSegment(cube);
- } else {
- endDate = Long.MAX_VALUE;
- }
- return appendSegments(cube, startDate, endDate, strictChecking, saveChange);
- }
-
- public CubeSegment appendSegments(CubeInstance cube, long startDate, long endDate, boolean strictChecking, boolean saveChange) throws IOException {
- if (strictChecking)
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment = newSegment(cube, startDate, endDate);
- validateNewSegments(cube, strictChecking, newSegment);
-
- if (saveChange) {
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToAddSegs(newSegment);
- updateCube(cubeBuilder);
- }
- return newSegment;
- }
-
- public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment = newSegment(cube, startDate, endDate);
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToAddSegs(newSegment);
- updateCube(cubeBuilder);
-
- return newSegment;
- }
-
- public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
- CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
-
- List<CubeSegment> mergingSegments = cube.getMergingSegments(newSegment);
-
- if (forceMergeEmptySeg == false) {
- List<String> emptySegment = Lists.newArrayList();
- for (CubeSegment seg : mergingSegments) {
- if (seg.getSizeKB() == 0) {
- emptySegment.add(seg.getName());
- }
- }
-
- if (emptySegment.size() > 0) {
- throw new IllegalArgumentException("Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment);
- }
- }
-
- validateNewSegments(cube, false, newSegment);
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToAddSegs(newSegment);
- updateCube(cubeBuilder);
-
- return newSegment;
- }
-
- private Pair<Long, Long> alignMergeRange(CubeInstance cube, long startDate, long endDate) {
- List<CubeSegment> readySegments = cube.getSegment(SegmentStatusEnum.READY);
- if (readySegments.isEmpty()) {
- throw new IllegalStateException("there are no segments in ready state");
- }
- long start = Long.MAX_VALUE;
- long end = Long.MIN_VALUE;
- for (CubeSegment readySegment : readySegments) {
- if (hasOverlap(startDate, endDate, readySegment.getDateRangeStart(), readySegment.getDateRangeEnd())) {
- if (start > readySegment.getDateRangeStart()) {
- start = readySegment.getDateRangeStart();
- }
- if (end < readySegment.getDateRangeEnd()) {
- end = readySegment.getDateRangeEnd();
- }
- }
- }
- return new Pair<Long, Long>(start, end);
- }
-
- private boolean hasOverlap(long startDate, long endDate, long anotherStartDate, long anotherEndDate) {
- if (startDate >= endDate) {
- throw new IllegalArgumentException("startDate must be less than endDate");
- }
- if (anotherStartDate >= anotherEndDate) {
- throw new IllegalArgumentException("anotherStartDate must be less than anotherEndDate");
- }
- if (startDate <= anotherStartDate && anotherStartDate < endDate) {
- return true;
- }
- if (startDate < anotherEndDate && anotherEndDate <= endDate) {
- return true;
- }
- return false;
- }
-
- private long calculateStartDateForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- } else {
- return existing.get(existing.size() - 1).getDateRangeEnd();
- }
- }
-
- private void checkNoBuildingSegment(CubeInstance cube) {
- if (cube.getBuildingSegments().size() > 0) {
- throw new IllegalStateException("There is already a building segment!");
- }
- }
-
- private void checkCubeIsPartitioned(CubeInstance cube) {
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) {
- throw new IllegalStateException("there is no partition date column specified, only full build is supported");
- }
- }
-
- /**
- * After cube update, reload cube related cache
- *
- * @param cubeName
- */
- public CubeInstance reloadCubeLocal(String cubeName) {
- try {
- return reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public void removeCubeLocal(String cubeName) {
- usedStorageLocation.removeAll(cubeName.toUpperCase());
- cubeMap.removeLocal(cubeName);
- }
-
- public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
-
- String tableName = dim.getTable();
- String[] pkCols = dim.getJoin().getPrimaryKey();
- String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
- if (snapshotResPath == null)
- throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
-
- try {
- SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
- return new LookupStringTable(tableDesc, pkCols, snapshot);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
- }
- }
-
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
- public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
- if (!cube.needAutoMerge()) {
- logger.debug("Cube " + cube.getName() + " doesn't need auto merge");
- return null;
- }
-
- if (cube.getBuildingSegments().size() > 0) {
- logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment");
- return null;
- }
-
- List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegment(SegmentStatusEnum.READY));
-
- if (readySegments.size() == 0) {
- logger.debug("Cube " + cube.getName() + " has no ready segment to merge");
- return null;
- }
-
- long[] timeRanges = cube.getAutoMergeTimeRanges();
- Arrays.sort(timeRanges);
-
- CubeSegment newSeg = null;
- for (int i = timeRanges.length - 1; i >= 0; i--) {
- long toMergeRange = timeRanges[i];
- long currentRange = 0;
- long lastEndTime = 0;
- List<CubeSegment> toMergeSegments = Lists.newArrayList();
- for (CubeSegment segment : readySegments) {
- long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart();
-
- if (thisSegmentRange >= toMergeRange) {
- // this segment and its previous segments will not be merged
- toMergeSegments.clear();
- currentRange = 0;
- lastEndTime = segment.getDateRangeEnd();
- continue;
- }
-
- if (segment.getDateRangeStart() != lastEndTime && toMergeSegments.isEmpty() == false) {
- // gap exists, give up the small segments before the gap;
- toMergeSegments.clear();
- currentRange = 0;
- }
-
- currentRange += thisSegmentRange;
- if (currentRange < toMergeRange) {
- toMergeSegments.add(segment);
- lastEndTime = segment.getDateRangeEnd();
- } else {
- // merge
- toMergeSegments.add(segment);
-
- newSeg = newSegment(cube, toMergeSegments.get(0).getDateRangeStart(), segment.getDateRangeEnd());
- // only one merge job be created here
- return newSeg;
- }
- }
-
- }
-
- return null;
- }
-
- public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
- List<CubeSegment> tobe = calculateToBeSegments(cube, false);
-
- for (CubeSegment seg : newSegments) {
- if (tobe.contains(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " is expected but not in the tobe " + tobe);
-
- if (StringUtils.isBlank(seg.getStorageLocationIdentifier()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing StorageLocationIdentifier");
-
- if (StringUtils.isBlank(seg.getLastBuildJobID()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing LastBuildJobID");
-
- seg.setStatus(SegmentStatusEnum.READY);
- }
-
- for (CubeSegment seg : tobe) {
- if (isReady(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " should be READY but is not");
- }
-
- List<CubeSegment> toRemoveSegs = Lists.newArrayList();
- for (CubeSegment segment : cube.getSegments()) {
- if (!tobe.contains(segment))
- toRemoveSegs.add(segment);
- }
-
- logger.info("Promoting cube " + cube + ", new segments " + Arrays.toString(newSegments) + ", to remove segments " + toRemoveSegs);
-
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToRemoveSegs(toRemoveSegs.toArray(new CubeSegment[toRemoveSegs.size()])).setToUpdateSegs(newSegments).setStatus(RealizationStatusEnum.READY);
- updateCube(cubeBuilder);
- }
-
- public void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
- validateNewSegments(cube, true, newSegments);
- }
-
- public void validateNewSegments(CubeInstance cube, boolean strictChecking, CubeSegment... newSegments) {
- List<CubeSegment> tobe = calculateToBeSegments(cube, strictChecking, newSegments);
- List<CubeSegment> newList = Arrays.asList(newSegments);
- if (tobe.containsAll(newList) == false) {
- throw new IllegalStateException("For cube " + cube + ", the new segments " + newList + " do not fit in its current " + cube.getSegments() + "; the resulted tobe is " + tobe);
- }
- }
-
- /**
- * Smartly figure out the TOBE segments once all new segments are built.
- * - Ensures no gap, no overlap
- * - Favors new segments over the old
- * - Favors big segments over the small
- */
- private List<CubeSegment> calculateToBeSegments(CubeInstance cube, boolean strictChecking, CubeSegment... newSegments) {
- CubeDesc cubeDesc = cube.getDescriptor();
- PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc();
-
- List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
- if (newSegments != null)
- tobe.addAll(Arrays.asList(newSegments));
- if (tobe.size() == 0)
- return tobe;
-
- // sort by start time, then end time
- Collections.sort(tobe);
-
- // check first segment start time
- CubeSegment firstSeg = tobe.get(0);
- firstSeg.validate();
-
- for (int i = 0, j = 1; j < tobe.size(); ) {
- CubeSegment is = tobe.get(i);
- CubeSegment js = tobe.get(j);
- js.validate();
-
- // check i is either ready or new
- if (!isNew(is) && !isReady(is)) {
- tobe.remove(i);
- continue;
- }
-
- // check j is either ready or new
- if (!isNew(js) && !isReady(js)) {
- tobe.remove(j);
- continue;
- }
-
- // if i, j competes
- if (is.getDateRangeStart() == js.getDateRangeStart()) {
- // if both new or ready, favor the bigger segment
- if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
- if (is.getDateRangeEnd() <= js.getDateRangeEnd()) {
- tobe.remove(i);
- } else {
- tobe.remove(j);
- }
- }
- // otherwise, favor the new segment
- else if (isNew(is)) {
- tobe.remove(j);
- } else {
- tobe.remove(i);
- }
- continue;
- }
-
- // if i, j in sequence
- if ((!strictChecking && is.getDateRangeEnd() <= js.getDateRangeStart() || strictChecking && is.getDateRangeEnd() == js.getDateRangeStart())) {
- i++;
- j++;
- continue;
- }
-
- // seems j not fitting
- tobe.remove(j);
- }
-
- return tobe;
- }
-
- private boolean isReady(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.READY;
- }
-
- private boolean isNew(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
- }
-
- private void loadAllCubeInstance() throws IOException {
- ResourceStore store = getStore();
- List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");
-
- logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
-
- for (String path : paths) {
- reloadCubeLocalAt(path);
- }
-
- logger.debug("Loaded " + paths.size() + " Cube(s)");
- }
-
- private synchronized CubeInstance reloadCubeLocalAt(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeInstance cubeInstance;
- try {
- cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
- cubeInstance.setConfig(config);
-
- if (StringUtils.isBlank(cubeInstance.getName()))
- throw new IllegalStateException("CubeInstance name must not be blank, at " + path);
-
- if (cubeInstance.getDescriptor() == null)
- throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path);
-
- final String cubeName = cubeInstance.getName();
- cubeMap.putLocal(cubeName, cubeInstance);
-
- for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier());
- }
-
- logger.info("Reloaded new cube: " + cubeName + " with reference being" + cubeInstance + " having " + cubeInstance.getSegments().size() + " segments:" +
- StringUtils.join(Collections2.transform(cubeInstance.getSegments(), new Function<CubeSegment, String>() {
- @Nullable
- @Override
- public String apply(CubeSegment input) {
- return input.getStorageLocationIdentifier();
- }
- }), ","));
-
- return cubeInstance;
- } catch (Exception e) {
- logger.error("Error during load cube instance " + path, e);
- return null;
- }
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private DictionaryManager getDictionaryManager() {
- return DictionaryManager.getInstance(config);
- }
-
- private SnapshotManager getSnapshotManager() {
- return SnapshotManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- @Override
- public RealizationType getRealizationType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public IRealization getRealization(String name) {
- return getCube(name);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
deleted file mode 100644
index a26c2c9..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.IDictionaryAware;
-import org.apache.kylin.metadata.model.IBuildable;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, IDictionaryAware, IBuildable {
-
- @JsonBackReference
- private CubeInstance cubeInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private SegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("input_records")
- private long inputRecords;
- @JsonProperty("input_records_size")
- private long inputRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of cube schema and dictionary ID, used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path
- @JsonProperty("snapshots")
- private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
-
- public CubeDesc getCubeDesc() {
- return getCubeInstance().getDescriptor();
- }
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public SegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(SegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getInputRecords() {
- return inputRecords;
- }
-
- public void setInputRecords(long inputRecords) {
- this.inputRecords = inputRecords;
- }
-
- public long getInputRecordsSize() {
- return inputRecordsSize;
- }
-
- public void setInputRecordsSize(long inputRecordsSize) {
- this.inputRecordsSize = inputRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public CubeInstance getCubeInstance() {
- return cubeInstance;
- }
-
- public void setCubeInstance(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- }
-
- public String getStorageLocationIdentifier() {
-
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Map<String, String> getSnapshots() {
- if (snapshots == null)
- snapshots = new ConcurrentHashMap<String, String>();
- return snapshots;
- }
-
- public String getSnapshotResPath(String table) {
- return getSnapshots().get(table);
- }
-
- public void putSnapshotResPath(String table, String snapshotResPath) {
- getSnapshots().put(table, snapshotResPath);
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public Collection<String> getSnapshotPaths() {
- return getSnapshots().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
- Dictionary<?> dict = getDictionary(col);
- if (dict == null) {
- return this.getCubeDesc().getRowkey().getColumnLength(col);
- } else {
- return dict.getSizeOfId();
- }
- }
-
- @Override
- public Dictionary<?> getDictionary(TblColRef col) {
- return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
- }
-
- public void validate() {
- if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned() && dateRangeStart >= dateRangeEnd)
- throw new IllegalStateException("dateRangeStart(" + dateRangeStart + ") must be smaller than dateRangeEnd(" + dateRangeEnd + ") in segment " + this);
- }
-
- @Override
- public int compareTo(CubeSegment other) {
- long comp = this.dateRangeStart - other.dateRangeStart;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
-
- comp = this.dateRangeEnd - other.dateRangeEnd;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
- else
- return 0;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((status == null) ? 0 : status.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CubeSegment other = (CubeSegment) obj;
- if (cubeInstance == null) {
- if (other.cubeInstance != null)
- return false;
- } else if (!cubeInstance.equals(other.cubeInstance))
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (status != other.status)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-
-
- public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {
- this.dictionaries = dictionaries;
- }
-
- public void setSnapshots(ConcurrentHashMap<String, String> snapshots) {
- this.snapshots = snapshots;
- }
-
- public String getStatisticsResourcePath() {
- return getStatisticsResourcePath(this.getCubeInstance().getName(), this.getUuid());
- }
-
- public static String getStatisticsResourcePath(String cubeName, String cubeSegmentId) {
- return ResourceStore.CUBE_STATISTICS_ROOT + "/" + cubeName + "/" + cubeSegmentId + ".seq";
- }
-
- @Override
- public int getSourceType() {
- return 0;
- }
-
- @Override
- public int getEngineType() {
- return 0;
- }
-
- @Override
- public int getStorageType() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7e8896ac/cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java b/cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
deleted file mode 100644
index dffaa48..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube;
-
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-
-/**
- * Hold changes to a cube so that they can be applied as one unit.
- */
-public class CubeUpdate {
- private CubeInstance cubeInstance;
- private CubeSegment[] toAddSegs = null;
- private CubeSegment[] toRemoveSegs = null;
- private CubeSegment[] toUpdateSegs = null;
- private RealizationStatusEnum status;
- private String owner;
- private int cost = -1;
-
- public CubeUpdate(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- }
-
- public CubeInstance getCubeInstance() {
- return cubeInstance;
- }
-
- public CubeUpdate setCubeInstance(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- return this;
- }
-
- public CubeSegment[] getToAddSegs() {
- return toAddSegs;
- }
-
- public CubeUpdate setToAddSegs(CubeSegment... toAddSegs) {
- this.toAddSegs = toAddSegs;
- return this;
- }
-
- public CubeSegment[] getToRemoveSegs() {
- return toRemoveSegs;
- }
-
- public CubeUpdate setToRemoveSegs(CubeSegment... toRemoveSegs) {
- this.toRemoveSegs = toRemoveSegs;
- return this;
- }
-
- public CubeSegment[] getToUpdateSegs() {
- return toUpdateSegs;
- }
-
- public CubeUpdate setToUpdateSegs(CubeSegment... toUpdateSegs) {
- this.toUpdateSegs = toUpdateSegs;
- return this;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public CubeUpdate setStatus(RealizationStatusEnum status) {
- this.status = status;
- return this;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public CubeUpdate setOwner(String owner) {
- this.owner = owner;
- return this;
- }
-
- public int getCost() {
- return cost;
- }
-
- public CubeUpdate setCost(int cost) {
- this.cost = cost;
- return this;
- }
-}