You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:50:19 UTC
[40/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
deleted file mode 100644
index 7420912..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescUpgrader.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.UUID;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.cube.model.HierarchyDesc;
-import org.apache.kylin.cube.model.RowKeyColDesc;
-import org.apache.kylin.cube.model.RowKeyDesc;
-import org.apache.kylin.cube.model.v1.CubeDesc;
-import org.apache.kylin.cube.model.v1.CubePartitionDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.JoinDesc;
-import org.apache.kylin.metadata.model.LookupDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class CubeDescUpgrader {
-
- private String resourcePath;
-
- @SuppressWarnings("unused")
- private static final Log logger = LogFactory.getLog(CubeDescUpgrader.class);
-
- private static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER_V1 = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- public CubeDescUpgrader(String resourcePath) {
- this.resourcePath = resourcePath;
- }
-
- public org.apache.kylin.cube.model.CubeDesc upgrade() throws IOException {
- CubeDesc oldModel = loadOldCubeDesc(resourcePath);
-
- org.apache.kylin.cube.model.CubeDesc newModel = new org.apache.kylin.cube.model.CubeDesc();
-
- copyUnChangedProperties(oldModel, newModel);
-
- DataModelDesc model = extractDataModel(oldModel, newModel);
- newModel.setModel(model);
-
- updateDimensions(oldModel, newModel);
-
- updateRowkeyDictionary(newModel);
-
- return newModel;
- }
-
- private void updateRowkeyDictionary(org.apache.kylin.cube.model.CubeDesc newModel) {
-
- DataModelDesc modelDesc = newModel.getModel();
- Map<String, String> pkToFK = Maps.newHashMap();
- for (LookupDesc lookupDesc : modelDesc.getLookups()) {
- if (lookupDesc.getJoin() != null) {
- JoinDesc join = lookupDesc.getJoin();
- for (int i = 0; i < join.getForeignKey().length; i++) {
- pkToFK.put(join.getPrimaryKey()[i], join.getForeignKey()[i]);
- }
- }
- }
- RowKeyDesc rowKey = newModel.getRowkey();
-
- for (RowKeyColDesc rowkeyCol : rowKey.getRowKeyColumns()) {
- if (rowkeyCol.getDictionary() != null && rowkeyCol.getDictionary().length() > 0)
- rowkeyCol.setDictionary("true");
-
- if (pkToFK.containsKey(rowkeyCol.getColumn())) {
- rowkeyCol.setColumn(pkToFK.get(rowkeyCol.getColumn()));
- }
- }
-
- for (String[] aggregationGroup : rowKey.getAggregationGroups()) {
- for (int i = 0; i < aggregationGroup.length; i++) {
- if (pkToFK.containsKey(aggregationGroup[i])) {
- aggregationGroup[i] = pkToFK.get(aggregationGroup[i]);
- }
- }
- }
-
- }
-
- private void copyUnChangedProperties(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- newModel.setUuid(oldModel.getUuid());
- newModel.setName(oldModel.getName());
- newModel.setDescription(oldModel.getDescription());
- newModel.setNullStrings(oldModel.getNullStrings());
-
- newModel.setMeasures(oldModel.getMeasures());
- newModel.setRowkey(oldModel.getRowkey());
- newModel.setHbaseMapping(oldModel.getHBaseMapping());
-
- newModel.setSignature(oldModel.getSignature());
-
- newModel.setNotifyList(oldModel.getNotifyList());
- newModel.setLastModified(oldModel.getLastModified());
- }
-
- private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) {
- DimensionDesc newDim = new DimensionDesc();
-
- newDim.setId(dimId);
- newDim.setName(name);
- newDim.setTable(getMetadataManager().appendDBName(dim.getTable()));
-
- return newDim;
- }
-
- private void updateDimensions(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<DimensionDesc> newDimensions = Lists.newArrayList();
- newModel.setDimensions(newDimensions);
-
- int dimId = 0;
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
-
- DimensionDesc newDim = null;
- // if a dimension defines "column", "derived" and "hierarchy" at the same time, separate it into three dimensions;
-
- boolean needNameSuffix = false;
- if (dim.getColumn() != null && !"{FK}".equals(dim.getColumn())) {
- //column on fact table
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setColumn(new String[] { dim.getColumn() });
- needNameSuffix = true;
- } else if (ArrayUtils.isEmpty(dim.getDerived()) && ArrayUtils.isEmpty(dim.getHierarchy())) {
- // user defines a lookup table, but didn't use any column other than the pk, in this case, convert to use fact table's fk
- newDim = newDimensionDesc(dim, dimId++, dim.getName());
- newDimensions.add(newDim);
- newDim.setTable(getMetadataManager().appendDBName(newModel.getFactTable()));
-
- newDim.setColumn(dim.getJoin().getForeignKey());
- }
-
- if (!ArrayUtils.isEmpty(dim.getDerived())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_DERIVED" : ""));
- newDimensions.add(newDim);
- newDim.setDerived(dim.getDerived());
- newDim.setColumn(null); // derived column must come from a lookup table; in this case the fk will be the dimension column, no need to explicitly declare it;
- needNameSuffix = true;
- }
-
- if (!ArrayUtils.isEmpty(dim.getHierarchy())) {
- newDim = newDimensionDesc(dim, dimId++, dim.getName() + (needNameSuffix ? "_HIERARCHY" : ""));
- newDimensions.add(newDim);
-
- newDim.setHierarchy(true);
-
- List<String> columns = Lists.newArrayList();
- for (HierarchyDesc hierarch : dim.getHierarchy()) {
- String col = hierarch.getColumn();
- columns.add(col);
- }
-
- newDim.setColumn(columns.toArray(new String[columns.size()]));
- }
-
- }
- }
-
- private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {
-
- DataModelDesc dm = new DataModelDesc();
- dm.setUuid(UUID.randomUUID().toString());
- String factTable = oldModel.getFactTable();
- dm.setName(oldModel.getName());
- dm.setFactTable(getMetadataManager().appendDBName(factTable));
-
- newModel.setModelName(dm.getName());
-
- List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();
-
- List<LookupDesc> lookups = Lists.newArrayList();
- List<String> lookupTables = Lists.newArrayList();
- for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
- JoinDesc join = dim.getJoin();
- if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) {
- String table = dim.getTable();
- table = getMetadataManager().appendDBName(table);
-
- if (lookupTables.contains(table)) {
- continue;
- }
- LookupDesc lookup = new LookupDesc();
- lookup.setJoin(join);
- lookup.setTable(table);
- lookupTables.add(table);
- lookups.add(lookup);
- }
- }
-
- dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
- dm.setFilterCondition(oldModel.getFilterCondition());
- updatePartitionDesc(oldModel, dm);
-
- if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM);
- } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) {
- dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE);
- }
-
- return dm;
- }
-
- private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) {
-
- CubePartitionDesc partition = oldModel.getCubePartitionDesc();
- PartitionDesc newPartition = new PartitionDesc();
-
- if (partition.getPartitionDateColumn() != null) {
- String partitionCol = partition.getPartitionDateColumn();
-
- String[] tablecolumn = partitionCol.split("\\.");
- if (tablecolumn != null && tablecolumn.length == 2) {
- // pattern is <tablename>.<colname>
- String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]);
- newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]);
- } else {
-
- if (partitionCol.indexOf(".") < 0) {
- // pattern is <colname>
- partitionCol = dm.getFactTable() + "." + partitionCol;
- }
-
- newPartition.setPartitionDateColumn(partitionCol);
- }
- }
-
- // only append is supported
- newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND);
-
- newPartition.setPartitionDateStart(partition.getPartitionDateStart());
-
- dm.setPartitionDesc(newPartition);
- }
-
- private CubeDesc loadOldCubeDesc(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER_V1);
-
- if (StringUtils.isBlank(ndesc.getName())) {
- throw new IllegalStateException("CubeDesc name must not be blank");
- }
-
- Map<String, TableDesc> tableMap = getMetadataManager().getAllTablesMap();
- Map<String, TableDesc> newMap = Maps.newHashMap();
- for (Entry<String, TableDesc> entry : tableMap.entrySet()) {
- String t = entry.getKey();
-
- if (t.indexOf(".") > 0) {
- newMap.put(t.substring(t.indexOf(".") + 1), entry.getValue());
-
- }
- }
- ndesc.init(KylinConfig.getInstanceFromEnv(), newMap);
-
- if (ndesc.getError().isEmpty() == false) {
- throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
- }
-
- return ndesc;
- }
-
- private static MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- }
-
- protected static ResourceStore getStore() {
- return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java b/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
deleted file mode 100644
index 8d771b9..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDimensionDeriver.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.Collection;
-import java.util.HashSet;
-
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * Created by Hongbin Ma(Binmahone) on 1/8/15.
- *
- * the unified logic for defining a sql's dimension
- */
-public class CubeDimensionDeriver {
-
- public static Collection<TblColRef> getDimensionColumns(Collection<TblColRef> groupByColumns, Collection<TblColRef> filterColumns) {
- Collection<TblColRef> dimensionColumns = new HashSet<TblColRef>();
- dimensionColumns.addAll(groupByColumns);
- dimensionColumns.addAll(filterColumns);
- return dimensionColumns;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
deleted file mode 100644
index 7773351..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeInstance extends RootPersistentEntity implements IRealization {
-
- public static CubeInstance create(String cubeName, String projectName, CubeDesc cubeDesc) {
- CubeInstance cubeInstance = new CubeInstance();
-
- cubeInstance.setConfig(cubeDesc.getConfig());
- cubeInstance.setName(cubeName);
- cubeInstance.setDescName(cubeDesc.getName());
- cubeInstance.setCreateTimeUTC(System.currentTimeMillis());
- cubeInstance.setSegments(new ArrayList<CubeSegment>());
- cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
- cubeInstance.updateRandomUuid();
- cubeInstance.setProjectName(projectName);
-
- return cubeInstance;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private RealizationStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<CubeSegment> segments = new ArrayList<CubeSegment>();
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- private String projectName;
-
- public List<CubeSegment> getBuildingSegments() {
- List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
- if (null != segments) {
- for (CubeSegment segment : segments) {
- if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
- buildingSegments.add(segment);
- }
- }
- }
-
- return buildingSegments;
- }
-
- public List<CubeSegment> getMergingSegments(CubeSegment mergedSegment) {
- List<CubeSegment> mergingSegments = new ArrayList<CubeSegment>();
- if (null != this.segments) {
- for (CubeSegment segment : this.segments) {
- if (!mergedSegment.equals(segment) //
- && mergedSegment.getDateRangeStart() <= segment.getDateRangeStart() && mergedSegment.getDateRangeEnd() >= segment.getDateRangeEnd()) {
- mergingSegments.add(segment);
- }
- }
- }
- return mergingSegments;
- }
-
- public CubeDesc getDescriptor() {
- return CubeDescManager.getInstance(config).getCubeDesc(descName);
- }
-
- public boolean isReady() {
- return getStatus() == RealizationStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.CUBE_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return getCanonicalName();
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("input_records_count")
- public long getInputRecordCount() {
- long sizeRecordCount = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getInputRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("input_records_size")
- public long getInputRecordSize() {
- long sizeRecordSize = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getCanonicalName() {
- return getType() + "[name=" + name + "]";
- }
-
- @Override
- public String getFactTable() {
- return this.getDescriptor().getFactTable();
- }
-
- @Override
- public List<MeasureDesc> getMeasures() {
- return getDescriptor().getMeasures();
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName.toUpperCase();
- }
-
- public String getOriginDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public void setCost(int cost) {
- this.cost = cost;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(RealizationStatusEnum status) {
- this.status = status;
- }
-
- public CubeSegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public CubeSegment getLatestReadySegment() {
- CubeSegment latest = null;
- for (int i = segments.size() - 1; i >= 0; i--) {
- CubeSegment seg = segments.get(i);
- if (seg.getStatus() != SegmentStatusEnum.READY)
- continue;
- if (latest == null || latest.getDateRangeEnd() < seg.getDateRangeEnd()) {
- latest = seg;
- }
- }
- return latest;
- }
-
- public List<CubeSegment> getSegments() {
- return segments;
- }
-
- public List<CubeSegment> getSegments(SegmentStatusEnum status) {
- List<CubeSegment> result = new ArrayList<CubeSegment>();
-
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public List<CubeSegment> getSegment(SegmentStatusEnum status) {
- List<CubeSegment> result = Lists.newArrayList();
- for (CubeSegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
- return result;
- }
-
- public CubeSegment getSegment(String name, SegmentStatusEnum status) {
- for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<CubeSegment> segments) {
- this.segments = segments;
- }
-
- public CubeSegment getSegmentById(String segmentId) {
- for (CubeSegment segment : segments) {
- if (Objects.equal(segment.getUuid(), segmentId)) {
- return segment;
- }
- }
- return null;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- @Override
- public CapabilityResult isCapable(SQLDigest digest) {
- CapabilityResult result = CubeCapabilityChecker.check(this, digest);
- if (result.capable) {
- result.cost = getCost(digest);
- for (CapabilityResult.CapabilityInfluence i : result.influences) {
- result.cost *= (i.suggestCostMultiplier() == 0) ? 1.0 : i.suggestCostMultiplier();
- }
- } else {
- result.cost = -1;
- }
- return result;
- }
-
- private int getCost(SQLDigest digest) {
- return cost;
- }
-
- @Override
- public RealizationType getType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return Lists.newArrayList(getDescriptor().listAllColumns());
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
-
- public boolean needAutoMerge() {
- if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
- return false;
-
- return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
- }
-
-
- @Override
- public long getDateRangeStart() {
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeStart() < startTime)
- startTime = seg.getDateRangeStart();
- }
-
- return startTime;
- }
-
- @Override
- public long getDateRangeEnd() {
-
- List<CubeSegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = 0;
- for (CubeSegment seg : readySegs) {
- if (seg.getDateRangeEnd() > endTime)
- endTime = seg.getDateRangeEnd();
- }
-
- return endTime;
- }
-
- @Override
- public List<TblColRef> getAllDimensions() {
- return Lists.newArrayList(getDescriptor().listDimensionColumnsIncludingDerived());
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
deleted file mode 100644
index d3bbf59..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ /dev/null
@@ -1,750 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.lookup.HiveTable;
-import org.apache.kylin.dict.lookup.LookupStringTable;
-import org.apache.kylin.dict.lookup.SnapshotManager;
-import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.IRealizationProvider;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
-
-/**
- * @author yangli9
- */
-public class CubeManager implements IRealizationProvider {
-
- private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
- private static int HBASE_TABLE_LENGTH = 10;
- public static final Serializer<CubeInstance> CUBE_SERIALIZER = new JsonSerializer<CubeInstance>(CubeInstance.class);
-
- private static final Logger logger = LoggerFactory.getLogger(CubeManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, CubeManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeManager>();
-
- public static CubeManager getInstance(KylinConfig config) {
- CubeManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one cubemanager singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // cube name ==> CubeInstance
- private CaseInsensitiveStringCache<CubeInstance> cubeMap = new CaseInsensitiveStringCache<CubeInstance>(Broadcaster.TYPE.CUBE);
- // "table/column" ==> lookup table
- // private SingleValueCache<String, LookupStringTable> lookupTables = new SingleValueCache<String, LookupStringTable>(Broadcaster.TYPE.METADATA);
-
- // for generation hbase table name of a new segment
- private Multimap<String, String> usedStorageLocation = HashMultimap.create();
-
- private CubeManager(KylinConfig config) throws IOException {
- logger.info("Initializing CubeManager with config " + config);
- this.config = config;
-
- loadAllCubeInstance();
- }
-
- public List<CubeInstance> listAllCubes() {
- return new ArrayList<CubeInstance>(cubeMap.values());
- }
-
- public CubeInstance getCube(String cubeName) {
- cubeName = cubeName.toUpperCase();
- return cubeMap.get(cubeName);
- }
-
- /**
- * Get related Cubes by cubedesc name. By default, the desc name will be
- * translated into upper case.
- *
- * @param descName CubeDesc name
- * @return
- */
- public List<CubeInstance> getCubesByDesc(String descName) {
-
- descName = descName.toUpperCase();
- List<CubeInstance> list = listAllCubes();
- List<CubeInstance> result = new ArrayList<CubeInstance>();
- Iterator<CubeInstance> it = list.iterator();
- while (it.hasNext()) {
- CubeInstance ci = it.next();
- if (descName.equals(ci.getDescName())) {
- result.add(ci);
- }
- }
- return result;
- }
-
- public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException {
- CubeDesc cubeDesc = cubeSeg.getCubeDesc();
- if (!cubeDesc.getRowkey().isUseDictionary(col))
- return null;
-
- DictionaryManager dictMgr = getDictionaryManager();
- DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
-
- if (dictInfo != null) {
- cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
- }
-
- return dictInfo;
- }
-
- /**
- * return null if no dictionary for given column
- */
- public Dictionary<String> getDictionary(CubeSegment cubeSeg, TblColRef col) {
- DictionaryInfo info = null;
- try {
- DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
- String dictResPath = cubeSeg.getDictResPath(col);
- if (dictResPath == null)
- return null;
-
- info = dictMgr.getDictionaryInfo(dictResPath);
- if (info == null)
- throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid cube state; cube segment" + cubeSeg + ", col " + col);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to get dictionary for cube segment" + cubeSeg + ", col" + col, e);
- }
-
- return (Dictionary<String>) info.getDictionaryObject();
- }
-
- public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
- MetadataManager metaMgr = getMetadataManager();
- SnapshotManager snapshotMgr = getSnapshotManager();
-
- HiveTable hiveTable = new HiveTable(metaMgr, lookupTable);
- TableDesc tableDesc = metaMgr.getTableDesc(lookupTable);
- SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);
-
- cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
-
- saveResource(cubeSeg.getCubeInstance());
-
- return snapshot;
- }
-
- // sync on update
- public CubeInstance dropCube(String cubeName, boolean deleteDesc) throws IOException {
- logger.info("Dropping cube '" + cubeName + "'");
- // load projects before remove cube from project
-
- ResourceStore store = getStore();
-
- // delete cube instance and cube desc
- CubeInstance cube = getCube(cubeName);
-
- if (deleteDesc && cube.getDescriptor() != null)
- store.deleteResource(cube.getDescriptor().getResourcePath());
-
- store.deleteResource(cube.getResourcePath());
-
- // delete cube from project
- ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
-
- // clean cube cache
- this.afterCubeDropped(cube);
-
- return cube;
- }
-
- // sync on update
- public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
- logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
-
- // save cube resource
- CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
- cube.setOwner(owner);
- saveResource(cube);
-
- ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
-
- return cube;
- }
-
- public CubeInstance updateCube(CubeInstance cube) throws IOException {
- logger.info("Updating cube instance '" + cube.getName());
- saveResource(cube);
- return cube;
- }
-
- public Pair<CubeSegment, CubeSegment> appendAndMergeSegments(CubeInstance cube, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- if (cube.getSegments().size() == 0)
- throw new IllegalStateException("expect at least one existing segment");
-
- long appendStart = calculateStartDateForAppendSegment(cube);
- CubeSegment appendSegment = newSegment(cube, appendStart, endDate);
-
- long startDate = cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- CubeSegment mergeSegment = newSegment(cube, startDate, endDate);
-
- validateNewSegments(cube, mergeSegment);
- cube.getSegments().add(appendSegment);
- cube.getSegments().add(mergeSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
- }
-
- public CubeSegment appendSegments(CubeInstance cube, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment;
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) {
- long startDate = calculateStartDateForAppendSegment(cube);
- newSegment = newSegment(cube, startDate, endDate);
- } else {
- newSegment = newSegment(cube, 0, Long.MAX_VALUE);
- }
-
- validateNewSegments(cube, newSegment);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
- checkNoBuildingSegment(cube);
-
- CubeSegment newSegment = newSegment(cube, startDate, endDate);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- public CubeSegment mergeSegments(CubeInstance cube, final long startDate, final long endDate, boolean forceMergeEmptySeg) throws IOException {
- checkNoBuildingSegment(cube);
- checkCubeIsPartitioned(cube);
-
- Pair<Long, Long> range = alignMergeRange(cube, startDate, endDate);
- CubeSegment newSegment = newSegment(cube, range.getFirst(), range.getSecond());
- List<CubeSegment> mergingSegments = cube.getMergingSegments(newSegment);
-
- if (forceMergeEmptySeg == false) {
- List<String> emptySegment = Lists.newArrayList();
- for (CubeSegment seg : mergingSegments) {
- if (seg.getSizeKB() == 0) {
- emptySegment.add(seg.getName());
- }
- }
- if (emptySegment.size() > 0) {
- throw new IllegalArgumentException("Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment);
- }
- }
-
- validateNewSegments(cube, newSegment);
- cube.getSegments().add(newSegment);
- Collections.sort(cube.getSegments());
- updateCube(cube);
-
- return newSegment;
- }
-
- private Pair<Long, Long> alignMergeRange(CubeInstance cube, long startDate, long endDate) {
- List<CubeSegment> readySegments = cube.getSegment(SegmentStatusEnum.READY);
- if (readySegments.isEmpty()) {
- throw new IllegalStateException("there are no segments in ready state");
- }
- long start = Long.MAX_VALUE;
- long end = Long.MIN_VALUE;
- for (CubeSegment readySegment : readySegments) {
- if (hasOverlap(startDate, endDate, readySegment.getDateRangeStart(), readySegment.getDateRangeEnd())) {
- if (start > readySegment.getDateRangeStart()) {
- start = readySegment.getDateRangeStart();
- }
- if (end < readySegment.getDateRangeEnd()) {
- end = readySegment.getDateRangeEnd();
- }
- }
- }
- return new Pair<Long, Long>(start, end);
- }
-
- private boolean hasOverlap(long startDate, long endDate, long anotherStartDate, long anotherEndDate) {
- if (startDate >= endDate) {
- throw new IllegalArgumentException("startDate must be less than endDate");
- }
- if (anotherStartDate >= anotherEndDate) {
- throw new IllegalArgumentException("anotherStartDate must be less than anotherEndDate");
- }
- if (startDate <= anotherStartDate && anotherStartDate < endDate) {
- return true;
- }
- if (startDate < anotherEndDate && anotherEndDate <= endDate) {
- return true;
- }
- return false;
- }
-
- private long calculateStartDateForAppendSegment(CubeInstance cube) {
- List<CubeSegment> existing = cube.getSegments();
- if (existing.isEmpty()) {
- return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart();
- } else {
- return existing.get(existing.size() - 1).getDateRangeEnd();
- }
- }
-
- private void checkNoBuildingSegment(CubeInstance cube) {
- if (cube.getBuildingSegments().size() > 0) {
- throw new IllegalStateException("There is already a building segment!");
- }
- }
-
- private void checkCubeIsPartitioned(CubeInstance cube) {
- if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) {
- throw new IllegalStateException("there is no partition date column specified, only full build is supported");
- }
- }
-
- public void updateSegmentOnJobDiscard(CubeInstance cubeInstance, String segmentName) throws IOException {
- for (int i = 0; i < cubeInstance.getSegments().size(); i++) {
- CubeSegment segment = cubeInstance.getSegments().get(i);
- if (segment.getName().equals(segmentName) && segment.getStatus() != SegmentStatusEnum.READY) {
- cubeInstance.getSegments().remove(segment);
- }
- }
- updateCube(cubeInstance);
- }
-
- /**
- * After cube update, reload cube related cache
- *
- * @param cubeName
- */
- public void loadCubeCache(String cubeName) {
- try {
- loadCubeInstance(CubeInstance.concatResourcePath(cubeName));
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
-
- /**
- * After cube deletion, remove cube related cache
- *
- * @param cube
- */
- public void removeCubeCache(CubeInstance cube) {
- final String cubeName = cube.getName().toUpperCase();
- cubeMap.remove(cubeName);
- usedStorageLocation.removeAll(cubeName);
- Cuboid.reloadCache(cube.getDescName());
- }
-
- public void removeCubeCacheLocal(String cubeName) {
- cubeMap.removeLocal(cubeName);
- usedStorageLocation.removeAll(cubeName);
- }
-
- public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
-
- String tableName = dim.getTable();
- String[] pkCols = dim.getJoin().getPrimaryKey();
- String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
- if (snapshotResPath == null)
- throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);
-
- try {
- SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
- TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
- return new LookupStringTable(tableDesc, pkCols, snapshot);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
- }
- }
-
- private void saveResource(CubeInstance cube) throws IOException {
- ResourceStore store = getStore();
- store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- this.afterCubeUpdated(cube);
- }
-
- private void afterCubeUpdated(CubeInstance updatedCube) {
- cubeMap.put(updatedCube.getName(), updatedCube);
- }
-
- private void afterCubeDropped(CubeInstance droppedCube) {
- removeCubeCache(droppedCube);
- }
-
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
- public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
- List<CubeSegment> tobe = calculateToBeSegments(cube);
-
- for (CubeSegment seg : newSegments) {
- if (tobe.contains(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " is expected but not in the tobe " + tobe);
-
- if (StringUtils.isBlank(seg.getStorageLocationIdentifier()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing StorageLocationIdentifier");
-
- if (StringUtils.isBlank(seg.getLastBuildJobID()))
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " missing LastBuildJobID");
-
- seg.setStatus(SegmentStatusEnum.READY);
- }
-
- for (CubeSegment seg : tobe) {
- if (isReady(seg) == false)
- throw new IllegalStateException("For cube " + cube + ", segment " + seg + " should be READY but is not");
- }
-
- cube.setSegments(tobe);
- cube.setStatus(RealizationStatusEnum.READY);
-
- logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- saveResource(cube);
- }
-
- private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
- List<CubeSegment> tobe = calculateToBeSegments(cube, newSegments);
- List<CubeSegment> newList = Arrays.asList(newSegments);
- if (tobe.containsAll(newList) == false) {
- throw new IllegalStateException("For cube " + cube + ", the new segments " + newList + " do not fit in its current " + cube.getSegments() + "; the resulted tobe is " + tobe);
- }
- }
-
- /**
- * Smartly figure out the TOBE segments once all new segments are built.
- * - Ensures no gap, no overlap
- * - Favors new segments over the old
- * - Favors big segments over the small
- */
- private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment... newSegments) {
- CubeDesc cubeDesc = cube.getDescriptor();
- PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc();
-
- List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
- if (newSegments != null)
- tobe.addAll(Arrays.asList(newSegments));
- if (tobe.size() == 0)
- return tobe;
-
- // sort by start time, then end time
- Collections.sort(tobe);
-
- // check first segment start time
- CubeSegment firstSeg = tobe.get(0);
- firstSeg.validate();
-
- for (int i = 0, j = 1; j < tobe.size();) {
- CubeSegment is = tobe.get(i);
- CubeSegment js = tobe.get(j);
- js.validate();
-
- // check i is either ready or new
- if (!isNew(is) && !isReady(is)) {
- tobe.remove(i);
- continue;
- }
-
- // check j is either ready or new
- if (!isNew(js) && !isReady(js)) {
- tobe.remove(j);
- continue;
- }
-
- // if i, j competes
- if (is.getDateRangeStart() == js.getDateRangeStart()) {
- // if both new or ready, favor the bigger segment
- if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
- if (is.getDateRangeEnd() <= js.getDateRangeEnd()) {
- tobe.remove(i);
- } else {
- tobe.remove(j);
- }
- }
- // otherwise, favor the new segment
- else if (isNew(is)) {
- tobe.remove(j);
- } else {
- tobe.remove(i);
- }
- continue;
- }
-
- // if i, j in sequence
- if (is.getDateRangeEnd() == js.getDateRangeStart()) {
- i++;
- j++;
- continue;
- }
-
- // seems j not fitting
- tobe.remove(j);
- }
-
- return tobe;
- }
-
- private boolean isReady(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.READY;
- }
-
- private boolean isNew(CubeSegment seg) {
- return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
- }
-
- private void loadAllCubeInstance() throws IOException {
- ResourceStore store = getStore();
- List<String> paths = store.collectResourceRecursively(ResourceStore.CUBE_RESOURCE_ROOT, ".json");
-
- logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
-
- for (String path : paths) {
- loadCubeInstance(path);
- }
-
- logger.debug("Loaded " + paths.size() + " Cube(s)");
- }
-
- private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
- ResourceStore store = getStore();
-
- CubeInstance cubeInstance;
- try {
- cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
- cubeInstance.setConfig(config);
-
- if (StringUtils.isBlank(cubeInstance.getName()))
- throw new IllegalStateException("CubeInstance name must not be blank, at " + path);
-
- if (cubeInstance.getDescriptor() == null)
- throw new IllegalStateException("CubeInstance desc not found '" + cubeInstance.getDescName() + "', at " + path);
-
- final String cubeName = cubeInstance.getName();
- cubeMap.putLocal(cubeName, cubeInstance);
-
- for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
- }
-
- return cubeInstance;
- } catch (Exception e) {
- logger.error("Error during load cube instance " + path, e);
- return null;
- }
- }
-
- public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
- if (!cube.needAutoMerge()) {
- logger.debug("Cube " + cube.getName() + " doesn't need auto merge");
- return null;
- }
-
- if (cube.getBuildingSegments().size() > 0) {
- logger.debug("Cube " + cube.getName() + " has bulding segment, will not trigger merge at this moment");
- return null;
- }
-
- List<CubeSegment> readySegments = Lists.newArrayList(cube.getSegment(SegmentStatusEnum.READY));
-
- if (readySegments.size() == 0) {
- logger.debug("Cube " + cube.getName() + " has no ready segment to merge");
- return null;
- }
-
- long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
- Arrays.sort(timeRanges);
-
- CubeSegment newSeg = null;
- for (int i = timeRanges.length - 1; i >= 0; i--) {
- long toMergeRange = timeRanges[i];
- long currentRange = 0;
- long lastEndTime = 0;
- List<CubeSegment> toMergeSegments = Lists.newArrayList();
- for (CubeSegment segment : readySegments) {
- long thisSegmentRange = segment.getDateRangeEnd() - segment.getDateRangeStart();
-
- if (thisSegmentRange >= toMergeRange) {
- // this segment and its previous segments will not be merged
- toMergeSegments.clear();
- currentRange = 0;
- lastEndTime = segment.getDateRangeEnd();
- continue;
- }
-
- if (segment.getDateRangeStart() != lastEndTime && toMergeSegments.isEmpty() == false) {
- // gap exists, give up the small segments before the gap;
- toMergeSegments.clear();
- currentRange = 0;
- }
-
- currentRange += thisSegmentRange;
- if (currentRange < toMergeRange) {
- toMergeSegments.add(segment);
- lastEndTime = segment.getDateRangeEnd();
- } else {
- // merge
- toMergeSegments.add(segment);
-
- newSeg = newSegment(cube, toMergeSegments.get(0).getDateRangeStart(), segment.getDateRangeEnd());
- // only one merge job be created here
- return newSeg;
- }
- }
-
- }
-
- return null;
- }
-
- private MetadataManager getMetadataManager() {
- return MetadataManager.getInstance(config);
- }
-
- private DictionaryManager getDictionaryManager() {
- return DictionaryManager.getInstance(config);
- }
-
- private SnapshotManager getSnapshotManager() {
- return SnapshotManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- @Override
- public RealizationType getRealizationType() {
- return RealizationType.CUBE;
- }
-
- @Override
- public IRealization getRealization(String name) {
- return getCube(name);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
deleted file mode 100644
index 12f0bf7..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-import com.google.common.collect.Maps;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class CubeSegment implements Comparable<CubeSegment>, ISegment {
-
- @JsonBackReference
- private CubeInstance cubeInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private SegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("input_records")
- private long inputRecords;
- @JsonProperty("input_records_size")
- private long inputRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of cube schema and dictionary ID, used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==> dictionary resource path
- @JsonProperty("snapshots")
- private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
-
- public CubeDesc getCubeDesc() {
- return getCubeInstance().getDescriptor();
- }
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public SegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(SegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getInputRecords() {
- return inputRecords;
- }
-
- public void setInputRecords(long inputRecords) {
- this.inputRecords = inputRecords;
- }
-
- public long getInputRecordsSize() {
- return inputRecordsSize;
- }
-
- public void setInputRecordsSize(long inputRecordsSize) {
- this.inputRecordsSize = inputRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public CubeInstance getCubeInstance() {
- return cubeInstance;
- }
-
- public void setCubeInstance(CubeInstance cubeInstance) {
- this.cubeInstance = cubeInstance;
- }
-
- public String getStorageLocationIdentifier() {
-
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Map<String, String> getSnapshots() {
- if (snapshots == null)
- snapshots = new ConcurrentHashMap<String, String>();
- return snapshots;
- }
-
- public String getSnapshotResPath(String table) {
- return getSnapshots().get(table);
- }
-
- public void putSnapshotResPath(String table, String snapshotResPath) {
- getSnapshots().put(table, snapshotResPath);
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public Collection<String> getSnapshotPaths() {
- return getSnapshots().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
- Dictionary<?> dict = getDictionary(col);
- if (dict == null) {
- return this.getCubeDesc().getRowkey().getColumnLength(col);
- } else {
- return dict.getSizeOfId();
- }
- }
-
- @Override
- public Dictionary<String> getDictionary(TblColRef col) {
- return CubeManager.getInstance(this.getCubeInstance().getConfig()).getDictionary(this, col);
- }
-
- public Map<TblColRef, Dictionary<String>> buildDictionaryMap() {
- Map<TblColRef, Dictionary<String>> result = Maps.newHashMap();
- for (TblColRef col : getCubeDesc().getAllColumnsNeedDictionary()) {
- result.put(col, (Dictionary<String>) getDictionary(col));
- }
- return result;
- }
-
-
- public void validate() {
- if (cubeInstance.getDescriptor().getModel().getPartitionDesc().isPartitioned() && dateRangeStart >= dateRangeEnd)
- throw new IllegalStateException("dateRangeStart(" + dateRangeStart + ") must be greater than dateRangeEnd(" + dateRangeEnd + ") in segment " + this);
- }
-
- @Override
- public int compareTo(CubeSegment other) {
- long comp = this.dateRangeStart - other.dateRangeStart;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
-
- comp = this.dateRangeEnd - other.dateRangeEnd;
- if (comp != 0)
- return comp < 0 ? -1 : 1;
- else
- return 0;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((cubeInstance == null) ? 0 : cubeInstance.hashCode());
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((status == null) ? 0 : status.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CubeSegment other = (CubeSegment) obj;
- if (cubeInstance == null) {
- if (other.cubeInstance != null)
- return false;
- } else if (!cubeInstance.equals(other.cubeInstance))
- return false;
- if (uuid == null) {
- if (other.uuid != null)
- return false;
- } else if (!uuid.equals(other.uuid))
- return false;
- if (name == null) {
- if (other.name != null)
- return false;
- } else if (!name.equals(other.name))
- return false;
- if (status != other.status)
- return false;
- return true;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-
- public void setDictionaries(ConcurrentHashMap<String, String> dictionaries) {
- this.dictionaries = dictionaries;
- }
-
- public void setSnapshots(ConcurrentHashMap<String, String> snapshots) {
- this.snapshots = snapshots;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
deleted file mode 100644
index 3cdaa93..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cli;
-
-import java.io.IOException;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.DimensionDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DictionaryGeneratorCLI {
-
- private static final Logger logger = LoggerFactory.getLogger(DictionaryGeneratorCLI.class);
-
- public static void processSegment(KylinConfig config, String cubeName, String segmentName, String factColumnsPath) throws IOException {
- CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
- CubeSegment segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
-
- processSegment(config, segment, factColumnsPath);
- }
-
- private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
- CubeManager cubeMgr = CubeManager.getInstance(config);
-
- for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
- // dictionary
- for (TblColRef col : dim.getColumnRefs()) {
- if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
- logger.info("Building dictionary for " + col);
- cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
- }
- }
-
- // build snapshot
- if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
- // CubeSegment seg = cube.getTheOnlySegment();
- logger.info("Building snapshot of " + dim.getTable());
- cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
- logger.info("Checking snapshot of " + dim.getTable());
- cubeMgr.getLookupTable(cubeSeg, dim); // load the table for
- // sanity check
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java b/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
deleted file mode 100644
index d4613e9..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/cli/DumpDictionaryCLI.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.cli;
-
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Date;
-
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryInfoSerializer;
-
-public class DumpDictionaryCLI {
-
- public static void main(String[] args) throws IOException {
- for (String path : args) {
- dump(new File(path));
- }
- }
-
- public static void dump(File f) throws IOException {
- if (f.isDirectory()) {
- for (File c : f.listFiles())
- dump(c);
- return;
- }
-
- if (f.getName().endsWith(".dict")) {
- DictionaryInfoSerializer ser = new DictionaryInfoSerializer();
- DictionaryInfo dictInfo = ser.deserialize(new DataInputStream(new FileInputStream(f)));
-
- System.out.println("============================================================================");
- System.out.println("File: " + f.getAbsolutePath());
- System.out.println(new Date(dictInfo.getLastModified()));
- System.out.println(JsonUtil.writeValueAsIndentString(dictInfo));
- dictInfo.getDictionaryObject().dump(System.out);
- System.out.println();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java b/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
deleted file mode 100644
index 4821c94..0000000
--- a/cube/src/main/java/org/apache/kylin/cube/common/RowKeySplitter.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.cube.common;
-
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.SplittedBytes;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.cuboid.Cuboid;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.cube.kv.RowKeyColumnIO;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author George Song (ysong1)
- *
- */
-public class RowKeySplitter {
-
- private CubeDesc cubeDesc;
- private RowKeyColumnIO colIO;
-
- private SplittedBytes[] splitBuffers;
- private int bufferSize;
-
- public SplittedBytes[] getSplitBuffers() {
- return splitBuffers;
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
- this.cubeDesc = cubeSeg.getCubeDesc();
- this.colIO = new RowKeyColumnIO(cubeSeg);
-
- this.splitBuffers = new SplittedBytes[splitLen];
- for (int i = 0; i < splitLen; i++) {
- this.splitBuffers[i] = new SplittedBytes(bytesLen);
- }
- this.bufferSize = 0;
- }
-
- /**
- * @param bytes
- * @param byteLen
- * @return cuboid ID
- */
- public long split(byte[] bytes, int byteLen) {
- this.bufferSize = 0;
- int offset = 0;
-
- // extract cuboid id
- SplittedBytes cuboidIdSplit = this.splitBuffers[this.bufferSize++];
- cuboidIdSplit.length = RowConstants.ROWKEY_CUBOIDID_LEN;
- System.arraycopy(bytes, offset, cuboidIdSplit.value, 0, RowConstants.ROWKEY_CUBOIDID_LEN);
- offset += RowConstants.ROWKEY_CUBOIDID_LEN;
-
- long cuboidId = Bytes.toLong(cuboidIdSplit.value, 0, cuboidIdSplit.length);
- Cuboid cuboid = Cuboid.findById(cubeDesc, cuboidId);
-
- // rowkey columns
- for (int i = 0; i < cuboid.getColumns().size(); i++) {
- TblColRef col = cuboid.getColumns().get(i);
- int colLength = colIO.getColumnLength(col);
- SplittedBytes split = this.splitBuffers[this.bufferSize++];
- split.length = colLength;
- System.arraycopy(bytes, offset, split.value, 0, colLength);
- offset += colLength;
- }
-
- return cuboidId;
- }
-}