You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:09:33 UTC
[48/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
new file mode 100644
index 0000000..03a6d6a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import net.jpountz.xxhash.XXHash32;
+import net.jpountz.xxhash.XXHashFactory;
+
+/**
+ * class that implements methods specific for dictionary data look up
+ */
+public class ColumnReverseDictionaryInfo extends AbstractColumnDictionaryInfo {
+
+ /**
+ * Map which will maintain mapping of byte array to surrogate key
+ */
+ private Map<DictionaryByteArrayWrapper, Integer> dictionaryByteArrayToSurrogateKeyMap;
+
+ /**
+ * hashing algorithm to calculate hash code
+ */
+ private XXHash32 xxHash32;
+
+ /**
+ * check and initialize xxHash32 if enabled
+ */
+ public ColumnReverseDictionaryInfo() {
+ boolean useXXHash = Boolean.valueOf(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.ENABLE_XXHASH,
+ CarbonCommonConstants.ENABLE_XXHASH_DEFAULT));
+ if (useXXHash) {
+ xxHash32 = XXHashFactory.fastestInstance().hash32();
+ }
+ }
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value as byte array. It will be treated as key here
+ * @return if found returns key else 0
+ */
+ @Override public int getSurrogateKey(byte[] value) {
+ DictionaryByteArrayWrapper dictionaryByteArrayWrapper =
+ new DictionaryByteArrayWrapper(value, xxHash32);
+ Integer surrogateKeyInMap =
+ dictionaryByteArrayToSurrogateKeyMap.get(dictionaryByteArrayWrapper);
+ if (null == surrogateKeyInMap) {
+ return CarbonCommonConstants.INVALID_SURROGATE_KEY;
+ }
+ return surrogateKeyInMap;
+ }
+
+ /**
+ * This method will add a new dictionary chunk to existing list of dictionary chunks
+ *
+ * @param dictionaryChunk
+ */
+ @Override public void addDictionaryChunk(List<byte[]> dictionaryChunk) {
+ dictionaryChunks.add(dictionaryChunk);
+ if (null == dictionaryByteArrayToSurrogateKeyMap) {
+ createDictionaryByteArrayToSurrogateKeyMap(dictionaryChunk.size());
+ }
+ addDataToDictionaryMap();
+ }
+
+ /**
+ * This method will add the new dictionary data to map
+ */
+ private void addDataToDictionaryMap() {
+ int surrogateKey = dictionaryByteArrayToSurrogateKeyMap.size();
+ List<byte[]> oneDictionaryChunk = dictionaryChunks.get(dictionaryChunks.size() - 1);
+ for (int i = 0; i < oneDictionaryChunk.size(); i++) {
+ // create a wrapper class that will calculate hash code for byte array
+ DictionaryByteArrayWrapper dictionaryByteArrayWrapper =
+ new DictionaryByteArrayWrapper(oneDictionaryChunk.get(i), xxHash32);
+ dictionaryByteArrayToSurrogateKeyMap.put(dictionaryByteArrayWrapper, ++surrogateKey);
+ }
+ }
+
+ /**
+ * This method will create the dictionary map. First time it will
+ * create dictionary map with capacity equal to list of byte arrays
+ *
+ * @param initialMapSize capacity to which map is to be instantiated
+ */
+ private void createDictionaryByteArrayToSurrogateKeyMap(int initialMapSize) {
+ dictionaryByteArrayToSurrogateKeyMap = new ConcurrentHashMap<>(initialMapSize);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
new file mode 100644
index 0000000..d29be86
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+/**
+ * dictionary interface which declares methods for finding surrogate key for a
+ * given dictionary value and finding dictionary value from a given surrogate key
+ */
+public interface Dictionary {
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value
+ * @return if found returns key else 0
+ */
+ int getSurrogateKey(String value);
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value as byte array
+ * @return if found returns key else -1
+ */
+ int getSurrogateKey(byte[] value);
+
+ /**
+ * This method will find and return the dictionary value for a given surrogate key.
+ * Applicable scenarios:
+ * 1. Query final result preparation : While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return value if found else null
+ */
+ String getDictionaryValueForKey(int surrogateKey);
+
+ /**
+ * This method will find and return the sort index for a given dictionary id.
+ * Applicable scenarios:
+ * 1. Used in case of order by queries when data sorting is required
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return if found returns key else 0
+ */
+ int getSortedIndex(int surrogateKey);
+
+ /**
+ * This method will find and return the dictionary value from sorted index.
+ * Applicable scenarios:
+ * 1. Query final result preparation in case of order by queries:
+ * While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param sortedIndex sort index of dictionary value
+ * @return value if found else null
+ */
+ String getDictionaryValueFromSortedIndex(int sortedIndex);
+
+ /**
+ * The method return the dictionary chunks wrapper of a column
+ * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+ * members.
+ * Applications Scenario:
+ * For preparing the column Sort info while writing the sort index file.
+ *
+ * @return
+ */
+ DictionaryChunksWrapper getDictionaryChunks();
+
+ /**
+ * This method will release the objects and set default value for primitive types
+ */
+ void clear();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
new file mode 100644
index 0000000..73f7702
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+import net.jpountz.xxhash.XXHash32;
+
+/**
+ * class that holds the byte array and overrides equals and hashcode method which
+ * will be useful for object comparison
+ */
+public class DictionaryByteArrayWrapper {
+
+ /**
+ * dictionary value as byte array
+ */
+ private byte[] data;
+
+ /**
+ * hashing algorithm to calculate hash code
+ */
+ private XXHash32 xxHash32;
+
+ /**
+ * @param data
+ */
+ public DictionaryByteArrayWrapper(byte[] data) {
+ this.data = data;
+ }
+
+ /**
+ * @param data
+ * @param xxHash32
+ */
+ public DictionaryByteArrayWrapper(byte[] data, XXHash32 xxHash32) {
+ this(data);
+ this.xxHash32 = xxHash32;
+ }
+
+ /**
+ * This method will compare 2 DictionaryByteArrayWrapper objects
+ *
+ * @param other
+ * @return
+ */
+ @Override public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+ DictionaryByteArrayWrapper otherObjectToCompare = (DictionaryByteArrayWrapper) other;
+ if (data.length != otherObjectToCompare.data.length) {
+ return false;
+ }
+ return ByteUtil.UnsafeComparer.INSTANCE.equals(data, otherObjectToCompare.data);
+
+ }
+
+ /**
+ * This method will calculate the hash code for given data
+ *
+ * @return
+ */
+ @Override public int hashCode() {
+ if (null != xxHash32) {
+ return xxHash32.hash(data, 0, data.length, 0);
+ }
+ int result = Arrays.hashCode(data);
+ result = 31 * result;
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
new file mode 100644
index 0000000..8da437b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+
+public interface DictionaryCacheLoader {
+
+ /**
+ * This method will load the dictionary data for a given columnIdentifier
+ *
+ * @param dictionaryInfo dictionary info object which will hold the required data
+ * for a given column
+ * @param columnIdentifier column unique identifier
+ * @param dictionaryChunkStartOffset start offset from where dictionary file has to
+ * be read
+ * @param dictionaryChunkEndOffset end offset till where dictionary file has to
+ * be read
+ * @param loadSortIndex flag to indicate whether the sort index file has to be
+ * read in memory after dictionary loading
+ * @throws IOException
+ */
+ void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
+ long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+ throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
new file mode 100644
index 0000000..6e603f9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.reader.CarbonDictionaryReader;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
+import org.apache.carbondata.core.service.DictionaryService;
+
+/**
+ * This class is responsible for loading the dictionary data for given columns
+ */
+public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
+
+ /**
+ * carbon table identifier
+ */
+ private CarbonTableIdentifier carbonTableIdentifier;
+
+ /**
+ * carbon store path
+ */
+ private String carbonStorePath;
+
+ /**
+ * @param carbonTableIdentifier fully qualified table name
+ * @param carbonStorePath hdfs store path
+ */
+ public DictionaryCacheLoaderImpl(CarbonTableIdentifier carbonTableIdentifier,
+ String carbonStorePath) {
+ this.carbonTableIdentifier = carbonTableIdentifier;
+ this.carbonStorePath = carbonStorePath;
+ }
+
+ /**
+ * This method will load the dictionary data for a given columnIdentifier
+ *
+ * @param dictionaryInfo dictionary info object which will hold the required data
+ * for a given column
+ * @param columnIdentifier column unique identifier
+ * @param dictionaryChunkStartOffset start offset from where dictionary file has to
+ * be read
+ * @param dictionaryChunkEndOffset end offset till where dictionary file has to
+ * be read
+ * @param loadSortIndex flag to indicate whether the sort index file has to be
+ * read in memory after dictionary loading
+ * @throws IOException
+ */
+ @Override public void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
+ long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+ throws IOException {
+ List<byte[]> dictionaryChunk =
+ load(columnIdentifier, dictionaryChunkStartOffset, dictionaryChunkEndOffset);
+ if (loadSortIndex) {
+ readSortIndexFile(dictionaryInfo, columnIdentifier);
+ }
+ dictionaryInfo.addDictionaryChunk(dictionaryChunk);
+ }
+
+ /**
+ * This method will load the dictionary data between a given start and end offset
+ *
+ * @param columnIdentifier column unique identifier
+ * @param startOffset start offset of dictionary file
+ * @param endOffset end offset of dictionary file
+ * @return list of dictionary value
+ * @throws IOException
+ */
+ private List<byte[]> load(ColumnIdentifier columnIdentifier, long startOffset, long endOffset)
+ throws IOException {
+ CarbonDictionaryReader dictionaryReader = getDictionaryReader(columnIdentifier);
+ List<byte[]> dictionaryValue = null;
+ try {
+ dictionaryValue = dictionaryReader.read(startOffset, endOffset);
+ } finally {
+ dictionaryReader.close();
+ }
+ return dictionaryValue;
+ }
+
+ /**
+ * This method will read the sort index file and load into memory
+ *
+ * @param dictionaryInfo
+ * @param columnIdentifier
+ * @throws IOException
+ */
+ private void readSortIndexFile(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier)
+ throws IOException {
+ CarbonDictionarySortIndexReader sortIndexReader = getSortIndexReader(columnIdentifier);
+ try {
+ dictionaryInfo.setSortOrderIndex(sortIndexReader.readSortIndex());
+ dictionaryInfo.setSortReverseOrderIndex(sortIndexReader.readInvertedSortIndex());
+ } finally {
+ sortIndexReader.close();
+ }
+ }
+
+ /**
+ * This method will create a dictionary reader instance to read the dictionary file
+ *
+ * @param columnIdentifier unique column identifier
+ * @return carbon dictionary reader instance
+ */
+ private CarbonDictionaryReader getDictionaryReader(ColumnIdentifier columnIdentifier) {
+ DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
+ return dictService
+ .getDictionaryReader(carbonTableIdentifier, columnIdentifier, carbonStorePath);
+ }
+
+ /**
+ * @param columnIdentifier unique column identifier
+ * @return sort index reader instance
+ */
+ private CarbonDictionarySortIndexReader getSortIndexReader(ColumnIdentifier columnIdentifier) {
+ DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
+ return dictService
+ .getDictionarySortIndexReader(carbonTableIdentifier, columnIdentifier, carbonStorePath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
new file mode 100644
index 0000000..c0c2096
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The wrapper class wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+ * members and expose the getSize API to get size of members in the List<List<byte>> chunks.
+ * Applications Scenario:
+ * For preparing the column Sort info while writing the sort index file.
+ */
+public class DictionaryChunksWrapper implements Iterator<byte[]> {
+
+ /**
+ * list of dictionaryChunks
+ */
+ private List<List<byte[]>> dictionaryChunks;
+
+ /**
+ * size of the list
+ */
+ private int size;
+
+ /**
+ * Current index of the list
+ */
+ private int currentIndex;
+
+ /**
+ * variable holds the count of elements already iterated
+ */
+ private int iteratorIndex;
+
+ /**
+ * variable holds the current index of List<List<byte[]>> being traversed
+ */
+ private int outerIndex;
+
+ /**
+ * Constructor of DictionaryChunksWrapper
+ *
+ * @param dictionaryChunks
+ */
+ public DictionaryChunksWrapper(List<List<byte[]>> dictionaryChunks) {
+ this.dictionaryChunks = dictionaryChunks;
+ for (List<byte[]> chunk : dictionaryChunks) {
+ this.size += chunk.size();
+ }
+ }
+
+ /**
+ * Returns {@code true} if the iteration has more elements.
+ * (In other words, returns {@code true} if {@link #next} would
+ * return an element rather than throwing an exception.)
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ return (currentIndex < size);
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ * The method pics the next elements from the first inner list till first is not finished, pics
+ * the second inner list ...
+ *
+ * @return the next element in the iteration
+ */
+ @Override public byte[] next() {
+ if (iteratorIndex >= dictionaryChunks.get(outerIndex).size()) {
+ iteratorIndex = 0;
+ outerIndex++;
+ }
+ byte[] value = dictionaryChunks.get(outerIndex).get(iteratorIndex);
+ currentIndex++;
+ iteratorIndex++;
+ return value;
+ }
+
+ /**
+ * Removes from the underlying collection the last element returned
+ * by this iterator (optional operation). This method can be called
+ * only once per call to {@link #next}. The behavior of an iterator
+ * is unspecified if the underlying collection is modified while the
+ * iteration is in progress in any way other than by calling this
+ * method.
+ *
+ * @throws UnsupportedOperationException if the {@code remove}
+ * operation is not supported by this iterator
+ * @throws IllegalStateException if the {@code next} method has not
+ * yet been called, or the {@code remove} method has already
+ * been called after the last call to the {@code next}
+ * method
+ * @implSpec The default implementation throws an instance of
+ * {@link UnsupportedOperationException} and performs no other action.
+ */
+ @Override public void remove() {
+ throw new UnsupportedOperationException("Remove operation not supported");
+ }
+
+ /**
+ * returns the total element size in List<List<byte[]>>
+ *
+ * @return
+ */
+ public int getSize() {
+ return size;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
new file mode 100644
index 0000000..c9b08dc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+
+/**
+ * dictionary column identifier which includes table identifier and column identifier
+ */
+public class DictionaryColumnUniqueIdentifier {
+
+ /**
+ * table fully qualified name
+ */
+ private CarbonTableIdentifier carbonTableIdentifier;
+
+ /**
+ * unique column id
+ */
+ private ColumnIdentifier columnIdentifier;
+
+ private DataType dataType;
+
+ /**
+ * Will be used in case of reverse dictionary cache which will be used
+ * in case of data loading.
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ */
+ public DictionaryColumnUniqueIdentifier(CarbonTableIdentifier carbonTableIdentifier,
+ ColumnIdentifier columnIdentifier) {
+ this.carbonTableIdentifier = carbonTableIdentifier;
+ this.columnIdentifier = columnIdentifier;
+ }
+
+ /**
+ * Will be used in case of forward dictionary cache in case
+ * of query execution.
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param dataType
+ */
+ public DictionaryColumnUniqueIdentifier(CarbonTableIdentifier carbonTableIdentifier,
+ ColumnIdentifier columnIdentifier, DataType dataType) {
+ this(carbonTableIdentifier, columnIdentifier);
+ this.dataType = dataType;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ /**
+ * @return table identifier
+ */
+ public CarbonTableIdentifier getCarbonTableIdentifier() {
+ return carbonTableIdentifier;
+ }
+
+ /**
+ * @return columnIdentifier
+ */
+ public ColumnIdentifier getColumnIdentifier() {
+ return columnIdentifier;
+ }
+
+ /**
+ * overridden equals method
+ *
+ * @param other
+ * @return
+ */
+ @Override public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other == null || getClass() != other.getClass()) return false;
+ DictionaryColumnUniqueIdentifier that = (DictionaryColumnUniqueIdentifier) other;
+ if (!carbonTableIdentifier.equals(that.carbonTableIdentifier)) return false;
+ return columnIdentifier.equals(that.columnIdentifier);
+
+ }
+
+ /**
+ * overridden hashcode method
+ *
+ * @return
+ */
+ @Override public int hashCode() {
+ int result = carbonTableIdentifier.hashCode();
+ result = 31 * result + columnIdentifier.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
new file mode 100644
index 0000000..e34860a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.List;
+
+import org.apache.carbondata.core.cache.Cacheable;
+
+/**
+ * An interface which holds dictionary information like end offset,
+ * file timestamp for one column
+ */
+public interface DictionaryInfo extends Cacheable, Dictionary {
+
+ /**
+ * This method will increment the access count for a column by 1
+ * whenever a column is getting used in query or incremental data load
+ */
+ void incrementAccessCount();
+
+ /**
+ * This method will update the end offset of file everytime a file is read
+ *
+ * @param offsetTillFileIsRead
+ */
+ void setOffsetTillFileIsRead(long offsetTillFileIsRead);
+
+ /**
+ * This method will update the timestamp of a file if a file is modified
+ * like in case of incremental load
+ *
+ * @param fileTimeStamp
+ */
+ void setFileTimeStamp(long fileTimeStamp);
+
+ /**
+ * This method will add a new dictionary chunk to existing list of dictionary chunks
+ *
+ * @param dictionaryChunk
+ */
+ void addDictionaryChunk(List<byte[]> dictionaryChunk);
+
+ /**
+ * This method will set the sort order index of a dictionary column.
+ * Sort order index if the index of dictionary values after they are sorted.
+ *
+ * @param sortOrderIndex
+ */
+ void setSortOrderIndex(List<Integer> sortOrderIndex);
+
+ /**
+ * This method will set the sort reverse index of a dictionary column.
+ * Sort reverse index is the index of dictionary values before they are sorted.
+ *
+ * @param sortReverseOrderIndex
+ */
+ void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex);
+
+ /**
+ * dictionary metadata file length which will be set whenever we reload dictionary
+ * data from disk
+ *
+ * @param dictionaryMetaFileLength length of dictionary metadata file
+ */
+ void setDictionaryMetaFileLength(long dictionaryMetaFileLength);
+
+ /**
+ * Dictionary meta file offset which will be read to check whether length of dictionary
+ * meta file has been modified
+ *
+ * @return
+ */
+ long getDictionaryMetaFileLength();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
new file mode 100644
index 0000000..bddab56
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * This class will be used for dictionary key and value look up
+ */
+public class ForwardDictionary implements Dictionary {
+
+ /**
+ * Object which will hold the information related to this dictionary column
+ */
+ private ColumnDictionaryInfo columnDictionaryInfo;
+
+ /**
+ * @param columnDictionaryInfo
+ */
+ public ForwardDictionary(ColumnDictionaryInfo columnDictionaryInfo) {
+ this.columnDictionaryInfo = columnDictionaryInfo;
+ }
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value
+ * @return if found returns key else 0
+ */
+ @Override public int getSurrogateKey(String value) {
+ return columnDictionaryInfo.getSurrogateKey(value);
+ }
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value as byte array
+ * @return if found returns key else 0
+ */
+ @Override public int getSurrogateKey(byte[] value) {
+ return columnDictionaryInfo.getSurrogateKey(value);
+ }
+
+ /**
+ * This method will find and return the dictionary value for a given surrogate key.
+ * Applicable scenarios:
+ * 1. Query final result preparation : While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return value if found else null
+ */
+ @Override public String getDictionaryValueForKey(int surrogateKey) {
+ return columnDictionaryInfo.getDictionaryValueForKey(surrogateKey);
+ }
+
+ /**
+ * This method will find and return the sort index for a given dictionary id.
+ * Applicable scenarios:
+ * 1. Used in case of order by queries when data sorting is required
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return if found returns key else 0
+ */
+ @Override public int getSortedIndex(int surrogateKey) {
+ return columnDictionaryInfo.getSortedIndex(surrogateKey);
+ }
+
+ /**
+ * This method will find and return the dictionary value from sorted index.
+ * Applicable scenarios:
+ * 1. Query final result preparation in case of order by queries:
+ * While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param sortedIndex sort index of dictionary value
+ * @return value if found else null
+ */
+ @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+ return columnDictionaryInfo.getDictionaryValueFromSortedIndex(sortedIndex);
+ }
+
+ /**
+ * The method return the dictionary chunks wrapper of a column
+ * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+ * members.
+ * Applications Scenario:
+ * For preparing the column Sort info while writing the sort index file.
+ *
+ * @return
+ */
+ @Override public DictionaryChunksWrapper getDictionaryChunks() {
+ return columnDictionaryInfo.getDictionaryChunks();
+ }
+
+ /**
+ * This method will release the objects and set default value for primitive types
+ */
+ @Override public void clear() {
+ if (null != columnDictionaryInfo) {
+ columnDictionaryInfo.clear();
+ columnDictionaryInfo = null;
+ }
+ }
+
+ /**
+ * This method will read the surrogates based on search range.
+ *
+ * @param surrogates
+ */
+ public void getSurrogateKeyByIncrementalSearch(List<String> evaluateResultList,
+ List<Integer> surrogates) {
+ List<byte[]> byteValuesOfFilterMembers = new ArrayList<byte[]>(evaluateResultList.size());
+ byte[] keyData = null;
+ for (int i = 0; i < evaluateResultList.size(); i++) {
+ keyData = evaluateResultList.get(i)
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ byteValuesOfFilterMembers.add(keyData);
+ }
+
+ columnDictionaryInfo
+ .getIncrementalSurrogateKeyFromDictionary(byteValuesOfFilterMembers, surrogates);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
new file mode 100644
index 0000000..d4f6adc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * This class implements methods to create dictionary cache which will hold
+ * dictionary chunks for look up of surrogate keys and values
+ */
+public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
+ V extends Dictionary>
+ extends AbstractDictionaryCache<K, V> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ForwardDictionaryCache.class.getName());
+
+ /**
+ * @param carbonStorePath
+ * @param carbonLRUCache
+ */
+ public ForwardDictionaryCache(String carbonStorePath, CarbonLRUCache carbonLRUCache) {
+ super(carbonStorePath, carbonLRUCache);
+ }
+
+ /**
+ * This method will get the value for the given key. If value does not exist
+ * for the given key, it will check and load the value.
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ @Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+ throws CarbonUtilException {
+ return getDictionary(dictionaryColumnUniqueIdentifier);
+ }
+
+ /**
+ * This method will return a list of values for the given list of keys.
+ * For each key, this method will check and load the data if required.
+ *
+ * @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return list of dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ @Override public List<Dictionary> getAll(
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
+ throws CarbonUtilException {
+ boolean exceptionOccurredInDictionaryLoading = false;
+ String exceptionMessage = "";
+ List<Dictionary> forwardDictionaryObjectList =
+ new ArrayList<Dictionary>(dictionaryColumnUniqueIdentifiers.size());
+ List<Future<Dictionary>> taskSubmitList =
+ new ArrayList<>(dictionaryColumnUniqueIdentifiers.size());
+ ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
+ for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
+ taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
+ @Override public Dictionary call() throws CarbonUtilException {
+ Dictionary dictionary = getDictionary(uniqueIdent);
+ return dictionary;
+ }
+ }));
+ }
+ try {
+ executorService.shutdown();
+ executorService.awaitTermination(2, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ LOGGER.error("Error loading the dictionary: " + e.getMessage());
+ }
+ for (int i = 0; i < taskSubmitList.size(); i++) {
+ try {
+ Dictionary columnDictionary = taskSubmitList.get(i).get();
+ forwardDictionaryObjectList.add(columnDictionary);
+ } catch (Throwable e) {
+ exceptionOccurredInDictionaryLoading = true;
+ exceptionMessage = e.getMessage();
+ }
+ }
+ if (exceptionOccurredInDictionaryLoading) {
+ clearDictionary(forwardDictionaryObjectList);
+ LOGGER.error(exceptionMessage);
+ throw new CarbonUtilException(exceptionMessage);
+ }
+ return forwardDictionaryObjectList;
+ }
+
+ /**
+ * This method will return the value for the given key. It will not check and load
+ * the data for the given key
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return
+ */
+ @Override public Dictionary getIfPresent(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ Dictionary forwardDictionary = null;
+ ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache.get(
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.FORWARD_DICTIONARY));
+ if (null != columnDictionaryInfo) {
+ forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
+ incrementDictionaryAccessCount(columnDictionaryInfo);
+ }
+ return forwardDictionary;
+ }
+
+ /**
+ * This method will remove the cache for a given key
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ */
+ @Override public void invalidate(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ carbonLRUCache.remove(
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.FORWARD_DICTIONARY));
+ }
+
+ /**
+ * This method will get the value for the given key. If value does not exist
+ * for the given key, it will check and load the value.
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ private Dictionary getDictionary(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+ throws CarbonUtilException {
+ Dictionary forwardDictionary = null;
+ // create column dictionary info object only if dictionary and its
+ // metadata file exists for a given column identifier
+ if (!isFileExistsForGivenColumn(dictionaryColumnUniqueIdentifier)) {
+ throw new CarbonUtilException(
+ "Either dictionary or its metadata does not exist for column identifier :: "
+ + dictionaryColumnUniqueIdentifier.getColumnIdentifier());
+ }
+ String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId();
+ ColumnDictionaryInfo columnDictionaryInfo =
+ getColumnDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier);
+ // load sort index file in case of forward dictionary
+ checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnDictionaryInfo,
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.FORWARD_DICTIONARY), true);
+ forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
+ return forwardDictionary;
+ }
+
+ /**
+ * This method will check and create columnDictionaryInfo object for the given column
+ *
+ * @param dictionaryColumnUniqueIdentifier
+ * @param columnIdentifier
+ * @return
+ */
+ private ColumnDictionaryInfo getColumnDictionaryInfo(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String columnIdentifier) {
+ ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache
+ .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY));
+ if (null == columnDictionaryInfo) {
+ synchronized (dictionaryColumnUniqueIdentifier) {
+ columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache
+ .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY));
+ if (null == columnDictionaryInfo) {
+ columnDictionaryInfo =
+ new ColumnDictionaryInfo(dictionaryColumnUniqueIdentifier.getDataType());
+ }
+ }
+ }
+ return columnDictionaryInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
new file mode 100644
index 0000000..1f8a3ff
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+/**
+ * This class will be used for dictionary key and value look up
+ */
+public class ReverseDictionary implements Dictionary {
+
+ /**
+ * Object which will hold the information related to this dictionary column
+ */
+ private ColumnReverseDictionaryInfo columnReverseDictionaryInfo;
+
+ /**
+ * @param columnReverseDictionaryInfo
+ */
+ public ReverseDictionary(ColumnReverseDictionaryInfo columnReverseDictionaryInfo) {
+ this.columnReverseDictionaryInfo = columnReverseDictionaryInfo;
+ }
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value
+ * @return if found returns key else 0
+ */
+ @Override public int getSurrogateKey(String value) {
+ return columnReverseDictionaryInfo.getSurrogateKey(value);
+ }
+
+ /**
+ * This method will find and return the surrogate key for a given dictionary value
+ * Applicable scenario:
+ * 1. Incremental data load : Dictionary will not be generated for existing values. For
+ * that values have to be looked up in the existing dictionary cache.
+ * 2. Filter scenarios where from value surrogate key has to be found.
+ *
+ * @param value dictionary value as byte array
+ * @return if found returns key else 0
+ */
+ @Override public int getSurrogateKey(byte[] value) {
+ return columnReverseDictionaryInfo.getSurrogateKey(value);
+ }
+
+ /**
+ * This method will find and return the dictionary value for a given surrogate key.
+ * Applicable scenarios:
+ * 1. Query final result preparation : While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return value if found else null
+ */
+ @Override public String getDictionaryValueForKey(int surrogateKey) {
+ return columnReverseDictionaryInfo.getDictionaryValueForKey(surrogateKey);
+ }
+
+ /**
+ * This method will find and return the sort index for a given dictionary id.
+ * Applicable scenarios:
+ * 1. Used in case of order by queries when data sorting is required
+ *
+ * @param surrogateKey a unique ID for a dictionary value
+ * @return if found returns key else 0
+ */
+ @Override public int getSortedIndex(int surrogateKey) {
+ return columnReverseDictionaryInfo.getSortedIndex(surrogateKey);
+ }
+
+ /**
+ * This method will find and return the dictionary value from sorted index.
+ * Applicable scenarios:
+ * 1. Query final result preparation in case of order by queries:
+ * While convert the final result which will
+ * be surrogate key back to original dictionary values this method will be used
+ *
+ * @param sortedIndex sort index of dictionary value
+ * @return value if found else null
+ */
+ @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+ return columnReverseDictionaryInfo.getDictionaryValueFromSortedIndex(sortedIndex);
+ }
+
+ /**
+ * The method return the dictionary chunks wrapper of a column
+ * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+ * members.
+ * Applications Scenario:
+ * For preparing the column Sort info while writing the sort index file.
+ *
+ * @return
+ */
+ @Override public DictionaryChunksWrapper getDictionaryChunks() {
+ return columnReverseDictionaryInfo.getDictionaryChunks();
+ }
+
+ /**
+ * This method will release the objects and set default value for primitive types
+ */
+ @Override public void clear() {
+ if (null != columnReverseDictionaryInfo) {
+ columnReverseDictionaryInfo.clear();
+ columnReverseDictionaryInfo = null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
new file mode 100644
index 0000000..aa05570
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * This class implements methods to create dictionary cache which will hold
+ * dictionary chunks for look up of surrogate keys and values
+ */
+public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
+ V extends Dictionary>
+ extends AbstractDictionaryCache<K, V> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ForwardDictionaryCache.class.getName());
+
+ /**
+ * @param carbonStorePath
+ * @param carbonLRUCache
+ */
+ public ReverseDictionaryCache(String carbonStorePath, CarbonLRUCache carbonLRUCache) {
+ super(carbonStorePath, carbonLRUCache);
+ }
+
+ /**
+ * This method will get the value for the given key. If value does not exist
+ * for the given key, it will check and load the value.
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ @Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+ throws CarbonUtilException {
+ return getDictionary(dictionaryColumnUniqueIdentifier);
+ }
+
+ /**
+ * This method will return a list of values for the given list of keys.
+ * For each key, this method will check and load the data if required.
+ *
+ * @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return list of dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ @Override public List<Dictionary> getAll(
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
+ throws CarbonUtilException {
+ boolean exceptionOccurredInDictionaryLoading = false;
+ String exceptionMessage = "";
+ List<Dictionary> reverseDictionaryObjectList =
+ new ArrayList<Dictionary>(dictionaryColumnUniqueIdentifiers.size());
+ List<Future<Dictionary>> taskSubmitList =
+ new ArrayList<>(dictionaryColumnUniqueIdentifiers.size());
+ ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
+ for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
+ taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
+ @Override public Dictionary call() throws CarbonUtilException {
+ Dictionary dictionary = getDictionary(uniqueIdent);
+ return dictionary;
+ }
+ }));
+ }
+ try {
+ executorService.shutdown();
+ executorService.awaitTermination(2, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ LOGGER.error("Error loading the dictionary: " + e.getMessage());
+ }
+ for (int i = 0; i < taskSubmitList.size(); i++) {
+ try {
+ Dictionary columnDictionary = taskSubmitList.get(i).get();
+ reverseDictionaryObjectList.add(columnDictionary);
+ } catch (Throwable e) {
+ exceptionOccurredInDictionaryLoading = true;
+ exceptionMessage = e.getMessage();
+ }
+ }
+ if (exceptionOccurredInDictionaryLoading) {
+ clearDictionary(reverseDictionaryObjectList);
+ LOGGER.error(exceptionMessage);
+ throw new CarbonUtilException(exceptionMessage);
+ }
+ return reverseDictionaryObjectList;
+ }
+
+ /**
+ * This method will return the value for the given key. It will not check and load
+ * the data for the given key
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return
+ */
+ @Override public Dictionary getIfPresent(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ Dictionary reverseDictionary = null;
+ ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+ (ColumnReverseDictionaryInfo) carbonLRUCache.get(
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.REVERSE_DICTIONARY));
+ if (null != columnReverseDictionaryInfo) {
+ reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo);
+ incrementDictionaryAccessCount(columnReverseDictionaryInfo);
+ }
+ return reverseDictionary;
+ }
+
+ /**
+ * This method will remove the cache for a given key
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ */
+ @Override public void invalidate(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+ carbonLRUCache.remove(
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.REVERSE_DICTIONARY));
+ }
+
+ /**
+ * This method will get the value for the given key. If value does not exist
+ * for the given key, it will check and load the value.
+ *
+ * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+ * tableName and columnIdentifier
+ * @return dictionary
+ * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+ */
+ private Dictionary getDictionary(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+ throws CarbonUtilException {
+ Dictionary reverseDictionary = null;
+ // create column dictionary info object only if dictionary and its
+ // metadata file exists for a given column identifier
+ if (!isFileExistsForGivenColumn(dictionaryColumnUniqueIdentifier)) {
+ throw new CarbonUtilException(
+ "Either dictionary or its metadata does not exist for column identifier :: "
+ + dictionaryColumnUniqueIdentifier.getColumnIdentifier());
+ }
+ String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId();
+ ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+ getColumnReverseDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier);
+ // do not load sort index file for reverse dictionary
+ checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnReverseDictionaryInfo,
+ getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+ CacheType.REVERSE_DICTIONARY), false);
+ reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo);
+ return reverseDictionary;
+ }
+
+ /**
+ * This method will check and create columnReverseDictionaryInfo object for the given column
+ *
+ * @param dictionaryColumnUniqueIdentifier
+ * @param columnIdentifier
+ * @return
+ */
+ private ColumnReverseDictionaryInfo getColumnReverseDictionaryInfo(
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String columnIdentifier) {
+ ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+ (ColumnReverseDictionaryInfo) carbonLRUCache
+ .get(getLruCacheKey(columnIdentifier, CacheType.REVERSE_DICTIONARY));
+ if (null == columnReverseDictionaryInfo) {
+ synchronized (dictionaryColumnUniqueIdentifier) {
+ columnReverseDictionaryInfo = (ColumnReverseDictionaryInfo) carbonLRUCache
+ .get(getLruCacheKey(columnIdentifier, CacheType.REVERSE_DICTIONARY));
+ if (null == columnReverseDictionaryInfo) {
+ columnReverseDictionaryInfo = new ColumnReverseDictionaryInfo();
+ }
+ }
+ }
+ return columnReverseDictionaryInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
new file mode 100644
index 0000000..c8f603a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+
+/**
+ * identifier which will have store path and carbon table identifier
+ */
+public class AbsoluteTableIdentifier implements Serializable {
+
+ /**
+ * serializable version
+ */
+ private static final long serialVersionUID = 4695047103484427506L;
+
+ /**
+ * path of the store
+ */
+ private String storePath;
+
+ /**
+ * carbon table identifier which will have table name and table database
+ * name
+ */
+ private CarbonTableIdentifier carbonTableIdentifier;
+
+ public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
+ //TODO this should be moved to common place where path handling will be handled
+ this.storePath = FileFactory.getUpdatedFilePath(storePath);
+ this.carbonTableIdentifier = carbonTableIdentifier;
+ }
+
+ /**
+ * @return the storePath
+ */
+ public String getStorePath() {
+ return storePath;
+ }
+
+ /**
+ * @return the carbonTableIdentifier
+ */
+ public CarbonTableIdentifier getCarbonTableIdentifier() {
+ return carbonTableIdentifier;
+ }
+
+ /**
+ * to get the hash code
+ */
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result =
+ prime * result + ((carbonTableIdentifier == null) ? 0 : carbonTableIdentifier.hashCode());
+ result = prime * result + ((storePath == null) ? 0 : storePath.hashCode());
+ return result;
+ }
+
+ /**
+ * to check this class is equal to
+ * other object passed
+ *
+ * @param obj other object
+ */
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof AbsoluteTableIdentifier)) {
+ return false;
+ }
+ AbsoluteTableIdentifier other = (AbsoluteTableIdentifier) obj;
+ if (carbonTableIdentifier == null) {
+ if (other.carbonTableIdentifier != null) {
+ return false;
+ }
+ } else if (!carbonTableIdentifier.equals(other.carbonTableIdentifier)) {
+ return false;
+ }
+ if (storePath == null) {
+ if (other.storePath != null) {
+ return false;
+ }
+ } else if (!storePath.equals(other.storePath)) {
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
new file mode 100644
index 0000000..7cfefc9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
+
+/**
+ * Wrapper Data Load Schema object which will be used to
+ * support relation while data loading
+ */
+public class CarbonDataLoadSchema implements Serializable {
+
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * CarbonTable info
+ */
+ private CarbonTable carbonTable;
+
+ /**
+ * dimension table and relation info
+ */
+ private List<DimensionRelation> dimensionRelationList;
+
+ /**
+ * CarbonDataLoadSchema constructor which takes CarbonTable
+ *
+ * @param carbonTable
+ */
+ public CarbonDataLoadSchema(CarbonTable carbonTable) {
+ this.carbonTable = carbonTable;
+ this.dimensionRelationList = new ArrayList<DimensionRelation>();
+ }
+
+ /**
+ * get dimension relation list
+ *
+ * @return dimensionRelationList
+ */
+ public List<DimensionRelation> getDimensionRelationList() {
+ return dimensionRelationList;
+ }
+
+ /**
+ * set dimensionrelation list
+ *
+ * @param dimensionRelationList
+ */
+ public void setDimensionRelationList(List<DimensionRelation> dimensionRelationList) {
+ this.dimensionRelationList = dimensionRelationList;
+ }
+
+ /**
+ * get carbontable
+ *
+ * @return carbonTable
+ */
+ public CarbonTable getCarbonTable() {
+ return carbonTable;
+ }
+
+ /**
+ * Dimension Relation object which will be filled from
+ * Load DML Command to support normalized table data load
+ */
+ public static class DimensionRelation implements Serializable {
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * dimension tableName
+ */
+ private String tableName;
+
+ /**
+ * dimensionSource csv path
+ */
+ private String dimensionSource;
+
+ /**
+ * relation with fact and dimension table
+ */
+ private Relation relation;
+
+ /**
+ * Columns to selected from dimension table.
+ * Hierarchy in-memory table should be prepared
+ * based on selected columns
+ */
+ private List<String> columns;
+
+ /**
+ * constructor
+ *
+ * @param tableName - dimension table name
+ * @param dimensionSource - source file path
+ * @param relation - fact foreign key with dimension primary key mapping
+ * @param columns - list of columns to be used from this dimension table
+ */
+ public DimensionRelation(String tableName, String dimensionSource, Relation relation,
+ List<String> columns) {
+ this.tableName = tableName;
+ this.dimensionSource = dimensionSource;
+ this.relation = relation;
+ this.columns = columns;
+ }
+
+ /**
+ * @return tableName
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * @return dimensionSource
+ */
+ public String getDimensionSource() {
+ return dimensionSource;
+ }
+
+ /**
+ * @return relation
+ */
+ public Relation getRelation() {
+ return relation;
+ }
+
+ /**
+ * @return columns
+ */
+ public List<String> getColumns() {
+ return columns;
+ }
+ }
+
+ /**
+ * Relation class to specify fact foreignkey column with
+ * dimension primary key column
+ */
+ public static class Relation implements Serializable {
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Fact foreign key column
+ */
+ private String factForeignKeyColumn;
+
+ /**
+ * dimension primary key column
+ */
+ private String dimensionPrimaryKeyColumn;
+
+ /**
+ * constructor
+ *
+ * @param factForeignKeyColumn - Fact Table Foreign key
+ * @param dimensionPrimaryKeyColumn - Dimension Table primary key
+ */
+ public Relation(String factForeignKeyColumn, String dimensionPrimaryKeyColumn) {
+ this.factForeignKeyColumn = factForeignKeyColumn;
+ this.dimensionPrimaryKeyColumn = dimensionPrimaryKeyColumn;
+ }
+
+ /**
+ * @return factForeignKeyColumn
+ */
+ public String getFactForeignKeyColumn() {
+ return factForeignKeyColumn;
+ }
+
+ /**
+ * @return dimensionPrimaryKeyColumn
+ */
+ public String getDimensionPrimaryKeyColumn() {
+ return dimensionPrimaryKeyColumn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
new file mode 100644
index 0000000..bb8a816
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+
+/**
+ * Identifier class which will hold the table qualified name
+ */
+public class CarbonTableIdentifier implements Serializable {
+
+ /**
+ * database name
+ */
+ private String databaseName;
+
+ /**
+ * table name
+ */
+ private String tableName;
+
+ /**
+ * table id
+ */
+ private String tableId;
+
+ /**
+ * constructor
+ */
+ public CarbonTableIdentifier(String databaseName, String tableName, String tableId) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.tableId = tableId;
+ }
+
+ /**
+ * return database name
+ */
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ /**
+ * return table name
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * @return tableId
+ */
+ public String getTableId() {
+ return tableId;
+ }
+
+ /**
+ * @return table unique name
+ */
+ public String getTableUniqueName() {
+ return databaseName + '_' + tableName;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((databaseName == null) ? 0 : databaseName.hashCode());
+ result = prime * result + ((tableId == null) ? 0 : tableId.hashCode());
+ result = prime * result + ((tableName == null) ? 0 : tableName.hashCode());
+ return result;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ CarbonTableIdentifier other = (CarbonTableIdentifier) obj;
+ if (databaseName == null) {
+ if (other.databaseName != null) {
+ return false;
+ }
+ } else if (!databaseName.equals(other.databaseName)) {
+ return false;
+ }
+ if (tableId == null) {
+ if (other.tableId != null) {
+ return false;
+ }
+ } else if (!tableId.equals(other.tableId)) {
+ return false;
+ }
+ if (tableName == null) {
+ if (other.tableName != null) {
+ return false;
+ }
+ } else if (!tableName.equals(other.tableName)) {
+ return false;
+ }
+ return true;
+ }
+
+ /*
+ * @return table unidque name
+ */
+ @Override public String toString() {
+ return databaseName + '_' + tableName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
new file mode 100644
index 0000000..9dda92c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+
+/**
+ * Column unique identifier
+ */
+public class ColumnIdentifier implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * column id
+ */
+ private String columnId;
+
+ /**
+ * column properties
+ */
+ private Map<String, String> columnProperties;
+
+ private DataType dataType;
+
+ /**
+ * @param columnId
+ * @param columnProperties
+ */
+ public ColumnIdentifier(String columnId, Map<String, String> columnProperties,
+ DataType dataType) {
+ this.columnId = columnId;
+ this.columnProperties = columnProperties;
+ this.dataType = dataType;
+ }
+
+ /**
+ * @return columnId
+ */
+ public String getColumnId() {
+ return columnId;
+ }
+
+ /**
+ * @param columnProperty
+ * @return
+ */
+ public String getColumnProperty(String columnProperty) {
+ if (null != columnProperties) {
+ return columnProperties.get(columnProperty);
+ }
+ return null;
+ }
+
+ public DataType getDataType() {
+ return this.dataType;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((columnId == null) ? 0 : columnId.hashCode());
+ return result;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnIdentifier other = (ColumnIdentifier) obj;
+ if (columnId == null) {
+ if (other.columnId != null) {
+ return false;
+ }
+ } else if (!columnId.equals(other.columnId)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override public String toString() {
+ return "ColumnIdentifier [columnId=" + columnId + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
new file mode 100644
index 0000000..9c090da
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
+
+/**
+ * below class holds the meta data requires to build the blocks
+ */
+public class BTreeBuilderInfo {
+
+ /**
+ * holds all the information about data
+ * file meta data
+ */
+ private List<DataFileFooter> footerList;
+
+ /**
+ * size of the each column value size
+ * this will be useful for reading
+ */
+ private int[] dimensionColumnValueSize;
+
+ public BTreeBuilderInfo(List<DataFileFooter> footerList,
+ int[] dimensionColumnValueSize) {
+ this.dimensionColumnValueSize = dimensionColumnValueSize;
+ this.footerList = footerList;
+ }
+
+ /**
+ * @return the eachDimensionBlockSize
+ */
+ public int[] getDimensionColumnValueSize() {
+ return dimensionColumnValueSize;
+ }
+
+ /**
+ * @return the footerList
+ */
+ public List<DataFileFooter> getFooterList() {
+ return footerList;
+ }
+}