You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/02/11 13:49:56 UTC
[17/51] [partial] kylin git commit: KYLIN-1416 keep only website in
document branch
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
deleted file mode 100644
index 86b0543..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIInstance.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.RootPersistentEntity;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.CapabilityResult;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.realization.SQLDigest;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonManagedReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * @author honma
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IIInstance extends RootPersistentEntity implements IRealization {
-
- public static IIInstance create(String iiName, String projectName, IIDesc iiDesc) {
- IIInstance iii = new IIInstance();
-
- iii.setConfig(iiDesc.getConfig());
- iii.setName(iiName);
- iii.setDescName(iiDesc.getName());
- iii.setCreateTimeUTC(System.currentTimeMillis());
- iii.setStatus(RealizationStatusEnum.DISABLED);
- iii.updateRandomUuid();
-
- return iii;
- }
-
- @JsonIgnore
- private KylinConfig config;
- @JsonProperty("name")
- private String name;
- @JsonProperty("owner")
- private String owner;
- @JsonProperty("version")
- private String version; // user info only, we don't do version control
- @JsonProperty("descriptor")
- private String descName;
- // Mark cube priority for query
- @JsonProperty("cost")
- private int cost = 50;
- @JsonProperty("status")
- private RealizationStatusEnum status;
-
- @JsonManagedReference
- @JsonProperty("segments")
- private List<IISegment> segments = new ArrayList<IISegment>();
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- private String projectName;
-
- public long getAllocatedEndDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(segments.size() - 1).getDateRangeEnd();
- }
-
- public long getAllocatedStartDate() {
- if (null == segments || segments.size() == 0) {
- return 0;
- }
-
- Collections.sort(segments);
-
- return segments.get(0).getDateRangeStart();
- }
-
- public IIDesc getDescriptor() {
- return IIDescManager.getInstance(config).getIIDesc(descName);
- }
-
- public boolean isReady() {
- return getStatus() == RealizationStatusEnum.READY;
- }
-
- public String getResourcePath() {
- return concatResourcePath(name);
- }
-
- public static String concatResourcePath(String cubeName) {
- return ResourceStore.II_RESOURCE_ROOT + "/" + cubeName + ".json";
- }
-
- @Override
- public String toString() {
- return getCanonicalName();
- }
-
- // ============================================================================
-
- @JsonProperty("size_kb")
- public long getSizeKB() {
- long sizeKb = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- return sizeKb;
- }
-
- @JsonProperty("input_records_count")
- public long getInputRecordCount() {
- long sizeRecordCount = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordCount += cubeSegment.getInputRecords();
- }
-
- return sizeRecordCount;
- }
-
- @JsonProperty("input_records_size")
- public long getInputRecordSize() {
- long sizeRecordSize = 0L;
-
- for (IISegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- return sizeRecordSize;
- }
-
- public KylinConfig getConfig() {
- return config;
- }
-
- public void setConfig(KylinConfig config) {
- this.config = config;
- }
-
- @Override
- public String getCanonicalName() {
- return getType() + "[name=" + name + "]";
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getDescName() {
- return descName;
- }
-
- public void setDescName(String descName) {
- this.descName = descName;
- }
-
- public int getCost() {
- return cost;
- }
-
- public RealizationStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(RealizationStatusEnum status) {
- this.status = status;
- }
-
- public IISegment getFirstSegment() {
- if (this.segments == null || this.segments.size() == 0) {
- return null;
- } else {
- return this.segments.get(0);
- }
- }
-
- public List<IISegment> getSegments() {
- return segments;
- }
-
- public List<IISegment> getSegments(SegmentStatusEnum status) {
- List<IISegment> result = new ArrayList<IISegment>();
-
- for (IISegment segment : segments) {
- if (segment.getStatus() == status) {
- result.add(segment);
- }
- }
-
- return result;
- }
-
- public IISegment getSegment(String name, SegmentStatusEnum status) {
- for (IISegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
- return segment;
- }
- }
-
- return null;
- }
-
- public void setSegments(List<IISegment> segments) {
- this.segments = segments;
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
- @Override
- public CapabilityResult isCapable(SQLDigest digest) {
- CapabilityResult result = IICapabilityChecker.check(this, digest);
- if (result.capable) {
- result.cost = getCost(digest);
- } else {
- result.cost = -1;
- }
- return result;
- }
-
- public int getCost(SQLDigest digest) {
- return 0;
- }
-
- @Override
- public RealizationType getType() {
- return RealizationType.INVERTED_INDEX;
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- return getDescriptor().listAllColumns();
- }
-
- @Override
- public String getFactTable() {
- return getDescriptor().getFactTableName();
- }
-
- @Override
- public List<MeasureDesc> getMeasures() {
- return getDescriptor().getMeasures();
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public void setProjectName(String projectName) {
- this.projectName = projectName;
- }
-
- @Override
- public long getDateRangeStart() {
- List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long startTime = Long.MAX_VALUE;
- for (IISegment seg : readySegs) {
- if (seg.getDateRangeStart() < startTime)
- startTime = seg.getDateRangeStart();
- }
-
- return startTime;
- }
-
- @Override
- public long getDateRangeEnd() {
-
- List<IISegment> readySegs = getSegments(SegmentStatusEnum.READY);
-
- long endTime = 0;
- for (IISegment seg : readySegs) {
- if (seg.getDateRangeEnd() > endTime)
- endTime = seg.getDateRangeEnd();
- }
-
- return endTime;
- }
-
- @Override
- public List<TblColRef> getAllDimensions() {
- return getDescriptor().listAllDimensions();
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
deleted file mode 100644
index f7e70f4..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.restclient.Broadcaster;
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.metadata.realization.IRealizationProvider;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author honma
- */
-public class IIManager implements IRealizationProvider {
-
- private static String ALPHA_NUM = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
- private static int HBASE_TABLE_LENGTH = 10;
-
- private static final Serializer<IIInstance> II_SERIALIZER = new JsonSerializer<IIInstance>(IIInstance.class);
-
- private static final Logger logger = LoggerFactory.getLogger(IIManager.class);
-
- // static cached instances
- private static final ConcurrentHashMap<KylinConfig, IIManager> CACHE = new ConcurrentHashMap<KylinConfig, IIManager>();
-
- public static IIManager getInstance(KylinConfig config) {
- IIManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (IIManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new IIManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init IIManager from " + config, e);
- }
- }
- }
-
- public static void clearCache() {
- CACHE.clear();
- }
-
- // ============================================================================
-
- private KylinConfig config;
- // ii name ==> IIInstance
- private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX);
-
- // for generation hbase table name of a new segment
- private HashSet<String> usedStorageLocation = new HashSet<String>();
-
- private IIManager(KylinConfig config) throws IOException {
- logger.info("Initializing IIManager with config " + config);
- this.config = config;
-
- loadAllIIInstance();
- }
-
- public List<IIInstance> listAllIIs() {
- return new ArrayList<IIInstance>(iiMap.values());
- }
-
- public IIInstance getII(String iiName) {
- iiName = iiName.toUpperCase();
- return iiMap.get(iiName);
- }
-
- public List<IIInstance> getIIsByDesc(String descName) {
-
- List<IIInstance> list = listAllIIs();
- List<IIInstance> result = new ArrayList<IIInstance>();
- Iterator<IIInstance> it = list.iterator();
- while (it.hasNext()) {
- IIInstance ci = it.next();
- if (descName.equalsIgnoreCase(ci.getDescName())) {
- result.add(ci);
- }
- }
- return result;
- }
-
- public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException {
- logger.info("Start building ii dictionary");
- DictionaryManager dictMgr = getDictionaryManager();
- IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor();
- for (TblColRef column : iiDesc.listAllColumns()) {
- logger.info("Dealing with column {}", column);
- if (iiDesc.isMetricsCol(column)) {
- continue;
- }
-
- DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
- iiSeg.putDictResPath(column, dict.getResourcePath());
- }
- saveResource(iiSeg.getIIInstance());
- }
-
- /**
- * return null if no dictionary for given column
- */
- public Dictionary<?> getDictionary(IISegment iiSeg, TblColRef col) {
- DictionaryInfo info = null;
- try {
- DictionaryManager dictMgr = getDictionaryManager();
- // logger.info("Using metadata url " + metadataUrl +
- // " for DictionaryManager");
- String dictResPath = iiSeg.getDictResPath(col);
- if (dictResPath == null)
- return null;
-
- info = dictMgr.getDictionaryInfo(dictResPath);
- if (info == null)
- throw new IllegalStateException("No dictionary found by " + dictResPath + ", invalid II state; II segment" + iiSeg + ", col " + col);
- } catch (IOException e) {
- throw new IllegalStateException("Failed to get dictionary for II segment" + iiSeg + ", col" + col, e);
- }
-
- return info.getDictionaryObject();
- }
-
- public IIInstance createII(IIInstance ii) throws IOException {
-
- if (this.getII(ii.getName()) != null)
- throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
-
- // other logic is the same as update.
- return updateII(ii);
- }
-
- public IIInstance updateII(IIInstance ii) throws IOException {
- logger.info("Updating II instance '" + ii.getName());
-
- // save resource
- saveResource(ii);
-
- logger.info("II with " + ii.getSegments().size() + " segments is saved");
-
- return ii;
- }
-
- public void loadIICache(String iiName) {
- try {
- loadIIInstance(IIInstance.concatResourcePath(iiName));
- } catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e);
- }
- }
-
- public void removeIICache(IIInstance ii) {
- iiMap.remove(ii.getName());
-
- for (IISegment segment : ii.getSegments()) {
- usedStorageLocation.remove(segment.getName());
- }
- }
-
- public void removeIILocalCache(String name) {
- iiMap.removeLocal(name);
- //TODO
- // for (IISegment segment : ii.getSegments()) {
- // usedStorageLocation.remove(segment.getName());
- // }
- }
-
- private void saveResource(IIInstance ii) throws IOException {
- ResourceStore store = getStore();
- store.putResource(ii.getResourcePath(), ii, II_SERIALIZER);
- this.afterIIUpdated(ii);
- }
-
- private void afterIIUpdated(IIInstance updatedII) {
- iiMap.put(updatedII.getName(), updatedII);
- }
-
- /**
- * @param IIInstance
- * @param startDate (pass 0 if full build)
- * @param endDate (pass 0 if full build)
- * @return
- */
- public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) {
- IISegment segment = new IISegment();
- String incrementalSegName = IISegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- segment.setCreateTimeUTC(System.currentTimeMillis());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setIIInstance(IIInstance);
-
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.IIHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- int idx = (int) (Math.random() * ALPHA_NUM.length());
- sb.append(ALPHA_NUM.charAt(idx));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.contains(tableName));
-
- return tableName;
- }
-
- private void loadAllIIInstance() throws IOException {
- ResourceStore store = getStore();
- List<String> paths = store.collectResourceRecursively(ResourceStore.II_RESOURCE_ROOT, ".json");
-
- logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));
-
- for (String path : paths) {
- loadIIInstance(path);
- }
-
- logger.debug("Loaded " + paths.size() + " II(s)");
- }
-
- private synchronized IIInstance loadIIInstance(String path) throws IOException {
- ResourceStore store = getStore();
- logger.debug("Loading IIInstance " + store.getReadableResourcePath(path));
-
- IIInstance IIInstance = null;
- try {
- IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER);
- IIInstance.setConfig(config);
-
- if (StringUtils.isBlank(IIInstance.getName()))
- throw new IllegalStateException("IIInstance name must not be blank");
-
- iiMap.putLocal(IIInstance.getName(), IIInstance);
-
- for (IISegment segment : IIInstance.getSegments()) {
- usedStorageLocation.add(segment.getName());
- }
-
- return IIInstance;
- } catch (Exception e) {
- logger.error("Error during load ii instance " + path, e);
- return null;
- }
- }
-
- private DictionaryManager getDictionaryManager() {
- return DictionaryManager.getInstance(config);
- }
-
- private ResourceStore getStore() {
- return ResourceStore.getStore(this.config);
- }
-
- @Override
- public RealizationType getRealizationType() {
- return RealizationType.INVERTED_INDEX;
- }
-
- @Override
- public IRealization getRealization(String name) {
- return getII(name);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
deleted file mode 100644
index 77a876c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IISegment.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex;
-
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.ISegment;
-import org.apache.kylin.invertedindex.index.TableRecordInfo;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonBackReference;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-
-/**
- * @author honma
- */
-
-// TODO: remove segment concept for II, append old hbase table
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
-public class IISegment implements Comparable<IISegment>, ISegment {
-
- @JsonBackReference
- private IIInstance iiInstance;
- @JsonProperty("uuid")
- private String uuid;
- @JsonProperty("name")
- private String name;
- @JsonProperty("storage_location_identifier")
- private String storageLocationIdentifier; // HTable name
- @JsonProperty("date_range_start")
- private long dateRangeStart;
- @JsonProperty("date_range_end")
- private long dateRangeEnd;
- @JsonProperty("status")
- private SegmentStatusEnum status;
- @JsonProperty("size_kb")
- private long sizeKB;
- @JsonProperty("input_records")
- private long inputRecords;
- @JsonProperty("input_records_size")
- private long inputRecordsSize;
- @JsonProperty("last_build_time")
- private long lastBuildTime;
- @JsonProperty("last_build_job_id")
- private String lastBuildJobID;
-
- @JsonProperty("create_time_utc")
- private long createTimeUTC;
-
- @JsonProperty("binary_signature")
- private String binarySignature; // a hash of schema and dictionary ID,
- // used for sanity check
-
- @JsonProperty("dictionaries")
- private ConcurrentHashMap<String, String> dictionaries; // table/column ==>
- // dictionary
- // resource path
-
- private transient TableRecordInfo tableRecordInfo;
-
- /**
- * @param startDate
- * @param endDate
- * @return if(startDate == 0 && endDate == 0), returns "FULL_BUILD", else
- * returns "yyyyMMddHHmmss_yyyyMMddHHmmss"
- */
- public static String getSegmentName(long startDate, long endDate) {
- if (startDate == 0 && endDate == 0) {
- return "FULL_BUILD";
- }
-
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss");
- dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
-
- return dateFormat.format(startDate) + "_" + dateFormat.format(endDate);
- }
-
- public IIDesc getIIDesc() {
- return getIIInstance().getDescriptor();
- }
-
- // ============================================================================
-
- public String getUuid() {
- return uuid;
- }
-
- public void setUuid(String id) {
- this.uuid = id;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public long getDateRangeStart() {
- return dateRangeStart;
- }
-
- public void setDateRangeStart(long dateRangeStart) {
- this.dateRangeStart = dateRangeStart;
- }
-
- public long getDateRangeEnd() {
- return dateRangeEnd;
- }
-
- public void setDateRangeEnd(long dateRangeEnd) {
- this.dateRangeEnd = dateRangeEnd;
- }
-
- public SegmentStatusEnum getStatus() {
- return status;
- }
-
- public void setStatus(SegmentStatusEnum status) {
- this.status = status;
- }
-
- public long getSizeKB() {
- return sizeKB;
- }
-
- public void setSizeKB(long sizeKB) {
- this.sizeKB = sizeKB;
- }
-
- public long getInputRecords() {
- return inputRecords;
- }
-
- public void setInputRecords(long inputRecords) {
- this.inputRecords = inputRecords;
- }
-
- public long getInputRecordsSize() {
- return inputRecordsSize;
- }
-
- public void setInputRecordsSize(long inputRecordsSize) {
- this.inputRecordsSize = inputRecordsSize;
- }
-
- public long getLastBuildTime() {
- return lastBuildTime;
- }
-
- public void setLastBuildTime(long lastBuildTime) {
- this.lastBuildTime = lastBuildTime;
- }
-
- public String getLastBuildJobID() {
- return lastBuildJobID;
- }
-
- public void setLastBuildJobID(String lastBuildJobID) {
- this.lastBuildJobID = lastBuildJobID;
- }
-
- public String getBinarySignature() {
- return binarySignature;
- }
-
- public void setBinarySignature(String binarySignature) {
- this.binarySignature = binarySignature;
- }
-
- public IIInstance getIIInstance() {
- return iiInstance;
- }
-
- public void setIIInstance(IIInstance iiInstance) {
- this.iiInstance = iiInstance;
- }
-
- public String getStorageLocationIdentifier() {
- return storageLocationIdentifier;
- }
-
- public Map<String, String> getDictionaries() {
- if (dictionaries == null)
- dictionaries = new ConcurrentHashMap<String, String>();
- return dictionaries;
- }
-
- public Collection<String> getDictionaryPaths() {
- return getDictionaries().values();
- }
-
- public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
- }
-
- public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
- }
-
- /**
- * @param storageLocationIdentifier the storageLocationIdentifier to set
- */
- public void setStorageLocationIdentifier(String storageLocationIdentifier) {
- this.storageLocationIdentifier = storageLocationIdentifier;
- }
-
- @Override
- public int compareTo(IISegment other) {
- if (this.dateRangeEnd < other.dateRangeEnd) {
- return -1;
- } else if (this.dateRangeEnd > other.dateRangeEnd) {
- return 1;
- } else {
- return 0;
- }
- }
-
- private TableRecordInfo getTableRecordInfo() {
- if (tableRecordInfo == null)
- tableRecordInfo = new TableRecordInfo(this);
- return tableRecordInfo;
- }
-
- public List<TblColRef> getColumns() {
- return this.getTableRecordInfo().getColumns();
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this).add("uuid", uuid).add("create_time_utc:", createTimeUTC).add("name", name).add("last_build_job_id", lastBuildJobID).add("status", status).toString();
- }
-
- @Override
- public int getColumnLength(TblColRef col) {
-
- int index = getTableRecordInfo().findColumn(col);
- return getTableRecordInfo().getDigest().length(index);
- }
-
- @Override
- public Dictionary<String> getDictionary(TblColRef col) {
-
- int index = getTableRecordInfo().findColumn(col);
- return getTableRecordInfo().dict(index);
- }
-
- public long getCreateTimeUTC() {
- return createTimeUTC;
- }
-
- public void setCreateTimeUTC(long createTimeUTC) {
- this.createTimeUTC = createTimeUTC;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
deleted file mode 100644
index 164e2b9..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public class BitMapContainer implements ColumnValueContainer {
-
- int valueLen;
- int nValues;
- int size;
- RoaringBitmap[] sets;
- boolean closedForChange;
-
- transient byte[] temp;
-
- public BitMapContainer(TableRecordInfoDigest digest, int col) {
- this.valueLen = digest.length(col);
- this.size = 0;
- this.nValues = digest.getMaxID(col) + 1;
- this.sets = null;
- this.closedForChange = false;
-
- this.temp = new byte[valueLen];
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen);
- append(value);
- }
-
- public void append(int value) {
- checkUpdateMode();
- if (value == Dictionary.NULL_ID[valueLen]) {
- value = nValues; // set[nValues] holds NULL
- }
- sets[value].add(size);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- int value = getValueIntAt(i);
- BytesUtil.writeUnsigned(value, temp, 0, valueLen);
- valueBytes.set(temp, 0, valueLen);
- }
-
- @Override
- public RoaringBitmap getBitMap(Integer startId, Integer endId) {
- if (startId == null && endId == null) {
- return sets[this.nValues];
- }
-
- int start = 0;
- int end = this.nValues - 1;
- if (startId != null) {
- start = startId;
- }
- if (endId != null) {
- end = endId;
- }
-
- return RoaringBitmap.or(Arrays.copyOfRange(sets, start, end + 1));
- }
-
- @SuppressWarnings("unused")
- private RoaringBitmap getBitMap(int valueId) {
- if (valueId >= 0 && valueId <= getMaxValueId())
- return sets[valueId];
- else
- return sets[this.nValues];
- }
-
- @Override
- public int getMaxValueId() {
- return this.nValues - 1;
- }
-
- public int getValueIntAt(int i) {
- for (int v = 0; v < nValues; v++) {
- if (sets[v].contains(i)) {
- return v;
- }
- }
- // if v is not in [0..nValues-1], then it must be nValue (NULL)
- return Dictionary.NULL_ID[valueLen];
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalStateException();
- }
- if (sets == null) {
- sets = new RoaringBitmap[nValues + 1];
- for (int i = 0; i <= nValues; i++) {
- sets[i] = new RoaringBitmap();
- }
- }
- }
-
- private boolean isClosedForChange() {
- return closedForChange;
- }
-
- @Override
- public void closeForChange() {
- closedForChange = true;
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public List<ImmutableBytesWritable> toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
-
- List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1);
- for (int i = 0; i <= nValues; i++) {
- r.add(setToBytes(sets[i]));
- }
- return r;
- }
-
- public void fromBytes(List<ImmutableBytesWritable> bytes) {
- assert nValues + 1 == bytes.size();
- sets = new RoaringBitmap[nValues + 1];
- size = 0;
- for (int i = 0; i <= nValues; i++) {
- sets[i] = bytesToSet(bytes.get(i));
- size += sets[i].getCardinality();
- }
- closedForChange = true;
- }
-
- private ImmutableBytesWritable setToBytes(RoaringBitmap set) {
- // Serializing a bitmap to a byte array can be expected to be expensive, this should not be commonly done.
- // If the purpose is to save the data to disk or to a network, then a direct serialization would be
- // far more efficient. If the purpose is to enforce immutability, it is an expensive way to do it.
- set.runOptimize(); //to improve compression
- final byte[] array = new byte[set.serializedSizeInBytes()];
- try {
- set.serialize(new java.io.DataOutputStream(new java.io.OutputStream() {
- int c = 0;
-
- @Override
- public void close() {
- }
-
- @Override
- public void flush() {
- }
-
- @Override
- public void write(int b) {
- array[c++] = (byte) b;
- }
-
- @Override
- public void write(byte[] b) {
- write(b, 0, b.length);
- }
-
- @Override
- public void write(byte[] b, int off, int l) {
- System.arraycopy(b, off, array, c, l);
- c += l;
- }
- }));
- } catch (IOException ioe) {
- // should never happen because we write to a byte array
- throw new RuntimeException("unexpected error while serializing to a byte array");
- }
-
- return new ImmutableBytesWritable(array);
- }
-
- private RoaringBitmap bytesToSet(final ImmutableBytesWritable bytes) {
- // converting a byte array to a bitmap can be expected to be expensive, hopefully this is not a common operation!
- RoaringBitmap set = new RoaringBitmap();
- if ((bytes.get() != null) && (bytes.getLength() > 0)) {
- // here we could use an ImmutableRoaringBitmap and just "map" it.
- // instead, we do a full deserialization
- // Note: we deserializing a Roaring bitmap, there is no need to know the length, the format is self-describing
- try {
- set.deserialize(new java.io.DataInputStream(new java.io.InputStream() {
- byte[] array = bytes.get();
- int c = bytes.getOffset();
-
- @Override
- public int read() {
- return array[c++] & 0xff;
- }
-
- @Override
- public int read(byte b[]) {
- return read(b, 0, b.length);
- }
-
- @Override
- public int read(byte[] b, int off, int l) {
- System.arraycopy(array, c, b, off, l);
- c += l;
- return l;
- }
- }));
- } catch (IOException ioe) {
- // should never happen because we read from a byte array
- throw new RuntimeException("unexpected error while deserializing from a byte array");
- }
- }
- return set;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + (closedForChange ? 1231 : 1237);
- result = prime * result + nValues;
- result = prime * result + Arrays.hashCode(sets);
- result = prime * result + size;
- result = prime * result + valueLen;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- BitMapContainer other = (BitMapContainer) obj;
- if (closedForChange != other.closedForChange)
- return false;
- if (nValues != other.nValues)
- return false;
- if (!Arrays.equals(sets, other.sets))
- return false;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
deleted file mode 100644
index a5ce1bd..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public interface ColumnValueContainer {
-
- void append(ImmutableBytesWritable valueBytes);
-
- void closeForChange();
-
- int getSize();
-
- // works only after closeForChange()
- void getValueAt(int i, ImmutableBytesWritable valueBytes);
-
- RoaringBitmap getBitMap(Integer startId, Integer endId);
-
- int getMaxValueId();
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
deleted file mode 100644
index 334457c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-
-import com.ning.compress.lzf.LZFDecoder;
-import com.ning.compress.lzf.LZFEncoder;
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * @author yangli9
- */
-public class CompressedValueContainer implements ColumnValueContainer {
- int valueLen;
- int nValues;
- int cap;
- int size;
- byte[] uncompressed;
- byte[] compressed;
-
- public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) {
- this.valueLen = digest.length(col);
- this.nValues = digest.getMaxID(col) + 1;
- this.cap = cap;
- this.size = 0;
- this.uncompressed = null;
- this.compressed = null;
- }
-
- @Override
- public void append(ImmutableBytesWritable valueBytes) {
- checkUpdateMode();
- System.arraycopy(valueBytes.get(), valueBytes.getOffset(), uncompressed, valueLen * size, valueLen);
- size++;
- }
-
- @Override
- public void getValueAt(int i, ImmutableBytesWritable valueBytes) {
- valueBytes.set(uncompressed, valueLen * i, valueLen);
- }
-
- @Override
- public RoaringBitmap getBitMap(Integer startId, Integer endId) {
- RoaringBitmap ret = new RoaringBitmap();
- int nullId = Dictionary.NULL_ID[valueLen];
-
- if (startId == null && endId == null) {
- //entry for getting null values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (nullId == valueID) {
- ret.add(i);
- }
- }
- return ret;
- }
-
- //normal values
- for (int i = 0; i < size; ++i) {
- int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen);
- if (valueID == nullId) {
- continue;
- }
-
- if (startId != null && valueID < startId) {
- continue;
- }
-
- if (endId != null && valueID > endId) {
- continue;
- }
-
- ret.add(i);
- }
- return ret;
-
- }
-
- @Override
- public int getMaxValueId() {
- return nValues - 1;
- }
-
- private void checkUpdateMode() {
- if (isClosedForChange()) {
- throw new IllegalArgumentException();
- }
- if (uncompressed == null) {
- uncompressed = new byte[valueLen * cap];
- }
- }
-
- private boolean isClosedForChange() {
- return compressed != null;
- }
-
- @Override
- public void closeForChange() {
- checkUpdateMode();
- try {
- compressed = LZFEncoder.encode(uncompressed, 0, valueLen * size);
- } catch (Exception e) {
- throw new RuntimeException("LZF encode failure", e);
- }
- }
-
- @Override
- public int getSize() {
- return size;
- }
-
- public ImmutableBytesWritable toBytes() {
- if (isClosedForChange() == false)
- closeForChange();
- return new ImmutableBytesWritable(compressed);
- }
-
- public void fromBytes(ImmutableBytesWritable bytes) {
- try {
- uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength());
- } catch (IOException e) {
- throw new RuntimeException("LZF decode failure", e);
- }
- size = cap = uncompressed.length / valueLen;
- compressed = BytesUtil.EMPTY_BYTE_ARRAY; // mark closed
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + size;
- result = prime * result + valueLen;
- result = prime * result + Arrays.hashCode(uncompressed);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- CompressedValueContainer other = (CompressedValueContainer) obj;
- if (size != other.size)
- return false;
- if (valueLen != other.valueLen)
- return false;
- if (!Bytes.equals(uncompressed, 0, size * valueLen, uncompressed, 0, size * valueLen))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
deleted file mode 100644
index b9f963e..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Arrays;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-
-/**
- * Created by honma on 11/10/14.
- */
-public class RawTableRecord implements Cloneable {
- TableRecordInfoDigest digest;
- private byte[] buf; // consecutive column value IDs (encoded by dictionary)
-
- public RawTableRecord(TableRecordInfoDigest info) {
- this.digest = info;
- this.buf = new byte[info.getByteFormLen()];
- reset();
- }
-
- public RawTableRecord(RawTableRecord another) {
- this.digest = another.digest;
- this.buf = Bytes.copy(another.buf);
- }
-
- public void reset() {
- Arrays.fill(buf, Dictionary.NULL);
- }
-
- public boolean isMetric(int col) {
- return digest.isMetrics(col);
- }
-
- public FixedLenMeasureCodec<LongWritable> codec(int col) {
- return digest.codec(col);
- }
-
- public int length(int col) {
- return digest.length(col);
- }
-
- public int getColumnCount() {
- return digest.getColumnCount();
- }
-
- public void setValueID(int col, int id) {
- BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col));
- }
-
- public int getValueID(int col) {
- return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col));
- }
-
- public void setValueMetrics(int col, LongWritable value) {
- digest.codec(col).write(value, buf, digest.offset(col));
- }
-
- public String getValueMetric(int col) {
- digest.codec(col).read(buf, digest.offset(col));
- return (String) digest.codec(col).getValue();
- }
-
- public byte[] getBytes() {
- return buf;
- }
-
- //TODO is it possible to avoid copying?
- public void setBytes(byte[] bytes, int offset, int length) {
- assert buf.length == length;
- System.arraycopy(bytes, offset, buf, 0, length);
- }
-
- public void setValueBytes(int col, ImmutableBytesWritable bytes) {
- System.arraycopy(bytes.get(), bytes.getOffset(), buf, digest.offset(col), digest.length(col));
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- bytes.set(buf, digest.offset(col), digest.length(col));
- }
-
- @Override
- public Object clone() {
- return new RawTableRecord(this);
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(buf);
- // result = prime * result + ((digest == null) ? 0 : digest.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- RawTableRecord other = (RawTableRecord) obj;
- if (!Arrays.equals(buf, other.buf))
- return false;
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
deleted file mode 100644
index 0fd1bf9..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingHash.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hashing;
-
-public class ShardingHash {
-
- static HashFunction hashFunc = Hashing.murmur3_128();
-
- public static long hashInt(int integer) {
- return hashFunc.newHasher().putInt(integer).hash().asLong();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
deleted file mode 100644
index 3443fc6..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-public class ShardingSliceBuilder {
-
- SliceBuilder[] builders;
-
- public ShardingSliceBuilder(TableRecordInfo info) {
- int sharding = info.getDescriptor().getSharding();
- builders = new SliceBuilder[sharding];
- for (short i = 0; i < sharding; i++) {
- builders[i] = new SliceBuilder(info, i);
- }
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- short shard = rec.getShard();
- return builders[shard].append(rec);
- }
-
- public List<Slice> close() {
- List<Slice> result = Lists.newArrayList();
- for (SliceBuilder builder : builders) {
- Slice slice = builder.close();
- if (slice != null)
- result.add(slice);
- }
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
deleted file mode 100644
index 2a53864..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-import org.roaringbitmap.RoaringBitmap;
-import org.roaringbitmap.IntIterator;
-
-/**
- * Within a partition (per timestampGranularity), records are further sliced
- * (per sliceLength) to fit into HBASE cell.
- *
- * @author yangli9
- */
-public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> {
-
- TableRecordInfoDigest info;
- int nColumns;
-
- short shard;
- long timestamp;
- int nRecords;
- ColumnValueContainer[] containers;
-
- public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) {
- this.info = digest;
- this.nColumns = digest.getColumnCount();
-
- this.shard = shard;
- this.timestamp = timestamp;
- this.nRecords = containers[0].getSize();
- this.containers = containers;
-
- assert nColumns == containers.length;
- for (int i = 0; i < nColumns; i++) {
- assert nRecords == containers[i].getSize();
- }
- }
-
- public int getRecordCount() {
- return this.nRecords;
- }
-
- public short getShard() {
- return shard;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public ColumnValueContainer[] getColumnValueContainers() {
- return containers;
- }
-
- public ColumnValueContainer getColumnValueContainer(int col) {
- return containers[col];
- }
-
- public Iterator<RawTableRecord> iterateWithBitmap(final RoaringBitmap resultBitMap) {
- if (resultBitMap == null) {
- return this.iterator();
- } else {
- final RawTableRecord rec = info.createTableRecordBytes();
- final ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- return new Iterator<RawTableRecord>() {
- IntIterator iter = resultBitMap.getIntIterator();
-
- @Override
- public boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public RawTableRecord next() {
- int i = iter.next();
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
- }
-
- @Override
- public Iterator<RawTableRecord> iterator() {
- return new Iterator<RawTableRecord>() {
- int i = 0;
- RawTableRecord rec = info.createTableRecordBytes();
- ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- @Override
- public boolean hasNext() {
- return i < nRecords;
- }
-
- @Override
- public RawTableRecord next() {
- for (int col = 0; col < nColumns; col++) {
- containers[col].getValueAt(i, temp);
- rec.setValueBytes(col, temp);
- }
- i++;
- return rec;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((info == null) ? 0 : info.hashCode());
- result = prime * result + shard;
- result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
- return result;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- Slice other = (Slice) obj;
- if (info == null) {
- if (other.info != null)
- return false;
- } else if (!info.equals(other.info))
- return false;
- if (shard != other.shard)
- return false;
- if (timestamp != other.timestamp)
- return false;
- return true;
- }
-
- @Override
- public int compareTo(Slice o) {
- int comp = this.shard - o.shard;
- if (comp != 0)
- return comp;
-
- comp = (int) (this.timestamp - o.timestamp);
- return comp;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
deleted file mode 100644
index d794708..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-
-/**
- * @author yangli9
- */
-public class SliceBuilder {
-
- TableRecordInfo info;
- private int nColumns;
- int nRecordsCap;
-
- short shard;
- long sliceTimestamp;
- int nRecords;
- private ColumnValueContainer[] containers;
-
- transient ImmutableBytesWritable temp = new ImmutableBytesWritable();
-
- public SliceBuilder(TableRecordInfo info, short shard) {
- this.info = info;
- this.nColumns = info.getDigest().getColumnCount();
- this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize());
-
- this.shard = shard;
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- this.containers = null;
-
- doneSlice(); // init containers
- }
-
- private Slice doneSlice() {
- Slice r = null;
- if (nRecords > 0) {
- for (int i = 0; i < nColumns; i++) {
- containers[i].closeForChange();
- }
- r = new Slice(info.getDigest(), shard, sliceTimestamp, containers);
- }
-
- // reset for next slice
- nRecords = 0;
- containers = new ColumnValueContainer[nColumns];
- for (int i : info.getDescriptor().getValueColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
- for (int i : info.getDescriptor().getMetricsColumns()) {
- containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap);
- }
-
- return r;
-
- }
-
- // NOTE: record must be appended in time order
- public Slice append(TableRecord rec) {
- if (rec.getShard() != shard)
- throw new IllegalStateException();
-
- Slice doneSlice = null;
-
- if (isFull()) {
- doneSlice = doneSlice();
- }
-
- if (nRecords == 0) {
- sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp());
- }
-
- nRecords++;
- for (int i = 0; i < nColumns; i++) {
- rec.getValueBytes(i, temp);
- containers[i].append(temp);
- }
-
- return doneSlice;
- }
-
- private long increaseSliceTimestamp(long timestamp) {
- if (timestamp < sliceTimestamp)
- throw new IllegalStateException();
-
- if (timestamp == sliceTimestamp)
- return ++timestamp; // ensure slice timestamp increases
- else
- return timestamp;
- }
-
- public Slice close() {
- Slice doneSlice = doneSlice();
- this.sliceTimestamp = Long.MIN_VALUE;
- this.nRecords = 0;
- return doneSlice;
- }
-
- private boolean isFull() {
- return nRecords >= nRecordsCap;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
deleted file mode 100644
index 09a9244..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.util.List;
-
-import org.apache.commons.lang.ObjectUtils;
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.DateFormat;
-
-import com.google.common.collect.Lists;
-
-/**
- * @author yangli9, honma
- * <p/>
- * TableRecord extends RawTableRecord by decorating it with a
- * TableRecordInfo
- */
-public class TableRecord implements Cloneable {
-
- private TableRecordInfo info;
- private RawTableRecord rawRecord;
-
- public TableRecord(RawTableRecord rawRecord, TableRecordInfo info) {
- this.info = info;
- this.rawRecord = rawRecord;
- }
-
- public TableRecord(TableRecord another) {
- this.info = another.info;
- this.rawRecord = (RawTableRecord) another.rawRecord.clone();
- }
-
- @Override
- public Object clone() {
- return new TableRecord(this);
- }
-
- public void reset() {
- rawRecord.reset();
- }
-
- public byte[] getBytes() {
- return rawRecord.getBytes();
- }
-
- public void setBytes(byte[] bytes, int offset, int length) {
- rawRecord.setBytes(bytes, offset, length);
- }
-
- public long getTimestamp() {
- String str = getValueString(info.getTimestampColumn());
- return DateFormat.stringToMillis(str);
- }
-
- public int length(int col) {
- return rawRecord.length(col);
- }
-
- public List<String> getOriginTableColumnValues() {
- List<String> ret = Lists.newArrayList();
- for (int i = 0; i < info.nColumns; ++i) {
- ret.add(getValueString(i));
- }
- return ret;
- }
-
- public void setValueString(int col, String value) {
- if (rawRecord.isMetric(col)) {
- LongWritable v = rawRecord.codec(col).valueOf(value);
- setValueMetrics(col, v);
- } else {
- int id = info.dict(col).getIdFromValue(value);
- rawRecord.setValueID(col, id);
- }
- }
-
- /**
- * get value of columns which belongs to the original table columns.
- * i.e. columns like min_xx, max_yy will never appear
- */
- public String getValueString(int col) {
- if (rawRecord.isMetric(col))
- return getValueMetric(col);
- else
- return info.dict(col).getValueFromId(rawRecord.getValueID(col));
- }
-
- public void getValueBytes(int col, ImmutableBytesWritable bytes) {
- rawRecord.getValueBytes(col, bytes);
- }
-
- private void setValueMetrics(int col, LongWritable value) {
- rawRecord.setValueMetrics(col, value);
- }
-
- private String getValueMetric(int col) {
- return rawRecord.getValueMetric(col);
- }
-
- public short getShard() {
- int timestampID = rawRecord.getValueID(info.getTimestampColumn());
- return (short) (Math.abs(ShardingHash.hashInt(timestampID)) % info.getDescriptor().getSharding());
- }
-
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder("[");
- for (int col = 0; col < rawRecord.getColumnCount(); col++) {
- if (col > 0)
- buf.append(",");
- buf.append(getValueString(col));
- }
- buf.append("]");
- return buf.toString();
- }
-
- @Override
- public int hashCode() {
- if (rawRecord != null) {
- return rawRecord.hashCode();
- } else {
- return 0;
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecord other = (TableRecord) obj;
- return ObjectUtils.equals(other.rawRecord, this.rawRecord);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
deleted file mode 100644
index c41a70c..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.kylin.common.util.Dictionary;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.invertedindex.IISegment;
-import org.apache.kylin.invertedindex.model.IIDesc;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-
-/**
- * @author yangli9
- * <p/>
- * TableRecordInfo stores application-aware knowledges, while
- * TableRecordInfoDigest only stores byte level knowleges
- */
-public class TableRecordInfo {
-
- final IISegment seg;
- final IIDesc desc;
- final int nColumns;
- final List<TblColRef> allColumns;
-
- final FixedLenMeasureCodec<?>[] measureSerializers;
- final Dictionary<?>[] dictionaries;
-
- final TableRecordInfoDigest digest;
-
- public TableRecordInfo(IISegment iiSegment) {
-
- seg = iiSegment;
- desc = seg.getIIInstance().getDescriptor();
- allColumns = desc.listAllColumns();
- nColumns = allColumns.size();
- dictionaries = new Dictionary<?>[nColumns];
- measureSerializers = new FixedLenMeasureCodec<?>[nColumns];
-
- DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig());
- int index = 0;
- for (TblColRef tblColRef : desc.listAllColumns()) {
- ColumnDesc col = tblColRef.getColumn();
- if (desc.isMetricsCol(index)) {
- measureSerializers[index] = FixedLenMeasureCodec.get(col.getType());
- } else {
- String dictPath = seg.getDictResPath(tblColRef);
- try {
- dictionaries[index] = dictMgr.getDictionary(dictPath);
- } catch (IOException e) {
- throw new RuntimeException("dictionary " + dictPath + " does not exist ", e);
- }
- }
- index++;
- }
-
- digest = createDigest();
- }
-
- public TableRecordInfoDigest getDigest() {
- return digest;
- }
-
- private TableRecordInfoDigest createDigest() {
- // isMetric
- boolean[] isMetric = new boolean[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- isMetric[i] = desc.isMetricsCol(i);
- }
-
- // lengths
- int[] lengths = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- lengths[i] = isMetric[i] ? measureSerializers[i].getLength() : dictionaries[i].getSizeOfId();
- }
-
- // dict max id
- int[] dictMaxIds = new int[nColumns];
- for (int i = 0; i < nColumns; ++i) {
- if (!isMetric[i])
- dictMaxIds[i] = dictionaries[i].getMaxId();
- }
-
- // offsets
- int pos = 0;
- int[] offsets = new int[nColumns];
- for (int i = 0; i < nColumns; i++) {
- offsets[i] = pos;
- pos += lengths[i];
- }
-
- int byteFormLen = pos;
-
- return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, measureSerializers);
- }
-
- public TableRecord createTableRecord() {
- return new TableRecord(digest.createTableRecordBytes(), this);
- }
-
- public IIDesc getDescriptor() {
- return desc;
- }
-
- public List<TblColRef> getColumns() {
- return allColumns;
- }
-
- public int findColumn(TblColRef col) {
- return desc.findColumn(col);
- }
-
- public int findFactTableColumn(String columnName) {
- if (columnName == null)
- return -1;
- for (int i = 0; i < allColumns.size(); ++i) {
- TblColRef tblColRef = allColumns.get(i);
- if (tblColRef.isSameAs(desc.getFactTableName(), columnName)) {
- return i;
- }
- }
- return -1;
- }
-
- // dimensions go with dictionary
- @SuppressWarnings("unchecked")
- public Dictionary<String> dict(int col) {
- // yes, all dictionaries are string based
- return (Dictionary<String>) dictionaries[col];
- }
-
- public int getTimestampColumn() {
- return desc.getTimestampColumn();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#hashCode()
- */
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((seg == null) ? 0 : seg.hashCode());
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- TableRecordInfo other = (TableRecordInfo) obj;
- if (seg == null) {
- if (other.seg != null)
- return false;
- } else if (!seg.equals(other.seg))
- return false;
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b6aa313/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
deleted file mode 100644
index 0ed58b0..0000000
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.invertedindex.index;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.kylin.common.util.BytesSerializer;
-import org.apache.kylin.common.util.BytesUtil;
-import org.apache.kylin.metadata.measure.fixedlen.FixedLenMeasureCodec;
-import org.apache.kylin.metadata.datatype.DataType;
-
-/**
- * Created by honma on 11/10/14.
- */
-public class TableRecordInfoDigest {
-
- private int nColumns;
- private int byteFormLen;
-
- private int[] offsets;// column offset in byte form row
- private int[] dictMaxIds;// max id for each of the dict
- private int[] lengths;// length of each encoded dict
- private boolean[] isMetric;// whether it's metric or dict
-
- protected FixedLenMeasureCodec<?>[] measureSerializers;
-
- public TableRecordInfoDigest(int nColumns, int byteFormLen, int[] offsets, int[] dictMaxIds, int[] lengths, boolean[] isMetric, FixedLenMeasureCodec<?>[] measureSerializers) {
- this.nColumns = nColumns;
- this.byteFormLen = byteFormLen;
- this.offsets = offsets;
- this.dictMaxIds = dictMaxIds;
- this.lengths = lengths;
- this.isMetric = isMetric;
- this.measureSerializers = measureSerializers;
- }
-
- public TableRecordInfoDigest() {
- }
-
- public int getByteFormLen() {
- return byteFormLen;
- }
-
- public boolean isMetrics(int col) {
- return isMetric[col];
- }
-
- public int getColumnCount() {
- return nColumns;
- }
-
- public int offset(int col) {
- return offsets[col];
- }
-
- public int length(int col) {
- return lengths[col];
- }
-
- public int getMaxID(int col) {
- return dictMaxIds[col];
- }
-
- public int getMetricCount() {
- int ret = 0;
- for (int i = 0; i < nColumns; ++i) {
- if (isMetrics(i)) {
- ret++;
- }
- }
- return ret;
- }
-
- public RawTableRecord createTableRecordBytes() {
- return new RawTableRecord(this);
- }
-
- // metrics go with fixed-len codec
- @SuppressWarnings("unchecked")
- public FixedLenMeasureCodec<LongWritable> codec(int col) {
- // yes, all metrics are long currently
- return (FixedLenMeasureCodec<LongWritable>) measureSerializers[col];
- }
-
- public static byte[] serialize(TableRecordInfoDigest o) {
- ByteBuffer buf = ByteBuffer.allocate(Serializer.SERIALIZE_BUFFER_SIZE);
- serializer.serialize(o, buf);
- byte[] result = new byte[buf.position()];
- System.arraycopy(buf.array(), 0, result, 0, buf.position());
- return result;
- }
-
- public static TableRecordInfoDigest deserialize(byte[] bytes) {
- return serializer.deserialize(ByteBuffer.wrap(bytes));
- }
-
- public static TableRecordInfoDigest deserialize(ByteBuffer buffer) {
- return serializer.deserialize(buffer);
- }
-
- private static final Serializer serializer = new Serializer();
-
- private static class Serializer implements BytesSerializer<TableRecordInfoDigest> {
-
- @Override
- public void serialize(TableRecordInfoDigest value, ByteBuffer out) {
- BytesUtil.writeVInt(value.nColumns, out);
- BytesUtil.writeVInt(value.byteFormLen, out);
- BytesUtil.writeIntArray(value.offsets, out);
- BytesUtil.writeIntArray(value.dictMaxIds, out);
- BytesUtil.writeIntArray(value.lengths, out);
- BytesUtil.writeBooleanArray(value.isMetric, out);
-
- for (int i = 0; i < value.measureSerializers.length; ++i) {
- if (value.isMetrics(i)) {
- BytesUtil.writeAsciiString(value.measureSerializers[i].getDataType().toString(), out);
- } else {
- BytesUtil.writeAsciiString(null, out);
- }
- }
- }
-
- @Override
- public TableRecordInfoDigest deserialize(ByteBuffer in) {
- TableRecordInfoDigest result = new TableRecordInfoDigest();
- result.nColumns = BytesUtil.readVInt(in);
- result.byteFormLen = BytesUtil.readVInt(in);
- result.offsets = BytesUtil.readIntArray(in);
- result.dictMaxIds = BytesUtil.readIntArray(in);
- result.lengths = BytesUtil.readIntArray(in);
- result.isMetric = BytesUtil.readBooleanArray(in);
-
- result.measureSerializers = new FixedLenMeasureCodec<?>[result.nColumns];
- for (int i = 0; i < result.nColumns; ++i) {
- String typeStr = BytesUtil.readAsciiString(in);
- if (typeStr == null) {
- result.measureSerializers[i] = null;
- } else {
- result.measureSerializers[i] = FixedLenMeasureCodec.get(DataType.getInstance(typeStr));
- }
- }
-
- return result;
- }
-
- }
-}