You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by jb...@apache.org on 2016/06/23 14:16:44 UTC
[56/56] [abbrv] incubator-carbondata git commit: Refactor carbon-core
module for code clean up (#720)
Refactor carbon-core module for code clean up (#720)
* refactor core module
* fix style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/63d3284e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/63d3284e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/63d3284e
Branch: refs/heads/master
Commit: 63d3284e4f0a09ef5248065da19c2bd3db371e08
Parents: d95742d
Author: Jacky Li <ja...@huawei.com>
Authored: Thu Jun 23 14:19:58 2016 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Thu Jun 23 11:49:58 2016 +0530
----------------------------------------------------------------------
.../org/carbondata/common/CarbonIterator.java | 38 ++
.../store/columnar/ColumnGroupModel.java | 137 +++++
.../store/columnar/ColumnarKeyStoreInfo.java | 1 -
.../store/columnar/UnBlockIndexer.java | 23 -
.../fileperations/AtomicFileOperations.java | 33 --
.../fileperations/AtomicFileOperationsImpl.java | 87 ----
.../store/fileperations/FileWriteOperation.java | 25 -
.../core/datastorage/util/StoreFactory.java | 143 +-----
.../core/file/manager/composite/FileData.java | 74 ---
.../file/manager/composite/FileManager.java | 68 ---
.../composite/IFileManagerComposite.java | 59 ---
.../core/iterator/CarbonIterator.java | 38 --
.../core/locks/AbstractCarbonLock.java | 77 ---
.../core/locks/CarbonLockFactory.java | 72 ---
.../org/carbondata/core/locks/HdfsFileLock.java | 89 ----
.../org/carbondata/core/locks/ICarbonLock.java | 40 --
.../carbondata/core/locks/LocalFileLock.java | 162 ------
.../org/carbondata/core/locks/LockUsage.java | 29 --
.../carbondata/core/locks/ZooKeeperLocking.java | 171 -------
.../carbondata/core/locks/ZookeeperInit.java | 64 ---
.../core/metadata/LevelNormalizedProps.java | 107 ----
.../carbondata/core/metadata/SliceMetaData.java | 333 -------------
.../carbondata/core/util/CarbonMergerUtil.java | 75 ---
.../core/util/CarbonSliceAndFiles.java | 88 ----
.../org/carbondata/core/util/CarbonUtil.java | 499 +------------------
.../carbondata/core/vo/ColumnGroupModel.java | 137 -----
.../core/writer/CarbonDataWriter.java | 482 ------------------
.../carbondata/scan/executor/QueryExecutor.java | 2 +-
.../scan/executor/impl/DetailQueryExecutor.java | 2 +-
.../impl/DetailRawRecordQueryExecutor.java | 2 +-
.../resolver/RestructureFilterResolverImpl.java | 8 -
.../processor/AbstractDataBlockIterator.java | 2 +-
.../scan/processor/BlockletIterator.java | 2 +-
.../org/carbondata/scan/result/BatchResult.java | 2 +-
.../AbstractDetailQueryResultIterator.java | 2 +-
.../scan/result/iterator/ChunkRowIterator.java | 2 +-
.../scan/result/iterator/RawResultIterator.java | 2 +-
.../core/locks/LocalFileLockTest.java | 48 --
.../core/locks/ZooKeeperLockingTest.java | 125 -----
.../carbondata/hadoop/CarbonRecordReader.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 6 +-
.../spark/merger/CarbonCompactionExecutor.java | 2 +-
.../spark/merger/RowResultMerger.java | 2 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 6 +-
.../execution/command/carbonTableSchema.scala | 3 +-
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 6 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 8 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../fileoperations/AtomicFileOperations.java | 33 ++
.../AtomicFileOperationsImpl.java | 87 ++++
.../lcm/fileoperations/FileWriteOperation.java | 25 +
.../lcm/locks/AbstractCarbonLock.java | 77 +++
.../carbondata/lcm/locks/CarbonLockFactory.java | 72 +++
.../org/carbondata/lcm/locks/HdfsFileLock.java | 89 ++++
.../org/carbondata/lcm/locks/ICarbonLock.java | 40 ++
.../org/carbondata/lcm/locks/LocalFileLock.java | 162 ++++++
.../org/carbondata/lcm/locks/LockUsage.java | 29 ++
.../carbondata/lcm/locks/ZooKeeperLocking.java | 171 +++++++
.../org/carbondata/lcm/locks/ZookeeperInit.java | 64 +++
.../lcm/status/SegmentStatusManager.java | 12 +-
.../processing/mdkeygen/MDKeyGenStep.java | 8 +-
.../processing/mdkeygen/file/FileData.java | 74 +++
.../processing/mdkeygen/file/FileManager.java | 68 +++
.../mdkeygen/file/IFileManagerComposite.java | 59 +++
.../merger/util/CarbonSliceMergerUtil.java | 366 --------------
.../store/CarbonFactDataHandlerColumnar.java | 6 +-
.../store/CarbonFactDataHandlerModel.java | 2 +-
.../store/colgroup/ColGroupDataHolder.java | 2 +-
.../store/colgroup/ColGroupMinMax.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
...actDataWriterImplForIntIndexAndAggBlock.java | 2 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 2 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 6 +-
.../FileStoreSurrogateKeyGenForCSV.java | 6 +-
.../carbondata/lcm/locks/LocalFileLockTest.java | 48 ++
.../lcm/locks/ZooKeeperLockingTest.java | 125 +++++
.../store/colgroup/ColGroupMinMaxTest.java | 7 +-
77 files changed, 1458 insertions(+), 3577 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/common/src/main/java/org/carbondata/common/CarbonIterator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/carbondata/common/CarbonIterator.java b/common/src/main/java/org/carbondata/common/CarbonIterator.java
new file mode 100644
index 0000000..2ac2ff3
--- /dev/null
+++ b/common/src/main/java/org/carbondata/common/CarbonIterator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.common;
+
+import java.util.Iterator;
+
+/**
+ * CarbonIterator adds default implement for remove. This is required for Java 7.
+ * @param <E>
+ */
+public abstract class CarbonIterator<E> implements Iterator<E> {
+
+ @Override public abstract boolean hasNext();
+
+ @Override public abstract E next();
+
+ @Override public void remove() {
+ throw new UnsupportedOperationException("remove");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..532c6e9
--- /dev/null
+++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.carbondata.core.datastorage.store.columnar;
+
+public class ColumnGroupModel {
+
+ /**
+ * cardinality all dimension
+ */
+ private int[] colGroupCardinality;
+
+ /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
+ * @return cardinality of column groups
+ */
+ public int[] getColumnGroupCardinality() {
+ return colGroupCardinality;
+ }
+
+ /**
+ * set columngroup cardinality
+ *
+ * @param columnGroupCardinality
+ */
+ public void setColumnGroupCardinality(int[] columnGroupCardinality) {
+ this.colGroupCardinality = columnGroupCardinality;
+ }
+
+ /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
index c80a887..0cf0d99 100644
--- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java
@@ -20,7 +20,6 @@
package org.carbondata.core.datastorage.store.columnar;
import org.carbondata.core.keygenerator.mdkey.NumberCompressor;
-import org.carbondata.core.vo.ColumnGroupModel;
public class ColumnarKeyStoreInfo {
private int numberOfKeys;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
index 26eb2ac..a20320b 100644
--- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
@@ -27,29 +27,6 @@ public final class UnBlockIndexer {
}
- public static short[] uncompressIndex(short[] indexData, short[] indexMap) {
- int actualSize = indexData.length;
- for (int i = 0; i < indexMap.length; i++) {
- actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
- }
- short[] indexes = new short[actualSize];
- int k = 0;
- for (short i = 0; i < indexData.length; i++) {
- int index = Arrays.binarySearch(indexMap, i);
- if (index > -1) {
- for (short j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
- indexes[k] = j;
- k++;
- }
- i++;
- } else {
- indexes[k] = indexData[i];
- k++;
- }
- }
- return indexes;
- }
-
public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
int actualSize = indexData.length;
for (int i = 0; i < indexMap.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java
deleted file mode 100644
index c5dcb29..0000000
--- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.datastorage.store.fileperations;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-public interface AtomicFileOperations {
-
- DataInputStream openForRead() throws IOException;
-
- void close() throws IOException;
-
- DataOutputStream openForWrite(FileWriteOperation operation) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java
deleted file mode 100644
index 41f4580..0000000
--- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.datastorage.store.fileperations;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-import org.carbondata.core.datastorage.store.impl.FileFactory.FileType;
-
-public class AtomicFileOperationsImpl implements AtomicFileOperations {
-
- private String filePath;
-
- private FileType fileType;
-
- private String tempWriteFilePath;
-
- private DataOutputStream dataOutStream;
-
- public AtomicFileOperationsImpl(String filePath, FileType fileType) {
- this.filePath = filePath;
-
- this.fileType = fileType;
- }
-
- @Override public DataInputStream openForRead() throws IOException {
- return FileFactory.getDataInputStream(filePath, fileType);
- }
-
- @Override public DataOutputStream openForWrite(FileWriteOperation operation) throws IOException {
-
- filePath = filePath.replace("\\", "/");
-
- tempWriteFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
-
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- FileFactory.getCarbonFile(tempWriteFilePath, fileType).delete();
- }
-
- FileFactory.createNewFile(tempWriteFilePath, fileType);
-
- dataOutStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
-
- return dataOutStream;
-
- }
-
- /* (non-Javadoc)
- * @see com.huawei.unibi.carbon.datastorage.store.fileperations.AtomicFileOperations#close()
- */
- @Override public void close() throws IOException {
-
- if (null != dataOutStream) {
- dataOutStream.close();
-
- CarbonFile tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
-
- if (!tempFile.renameForce(filePath)) {
- throw new IOException("temporary file renaming failed, src="
- + tempFile.getPath() + ", dest=" + filePath);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java
deleted file mode 100644
index 58026d9..0000000
--- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.datastorage.store.fileperations;
-
-public enum FileWriteOperation {
-
- APPEND, OVERWRITE
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
index 725ee56..8fc5aac 100644
--- a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
+++ b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java
@@ -20,63 +20,28 @@
package org.carbondata.core.datastorage.util;
import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.FileHolder;
-import org.carbondata.core.datastorage.store.NodeKeyStore;
import org.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStore;
-import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo;
import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataFileStore;
import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataFileStore;
+
import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarFileKeyStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarFileKeyStore;
-import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyFileStore;
-import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyInMemoryStore;
-import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyFileStore;
-import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyInMemoryStore;
import org.carbondata.core.util.CarbonProperties;
public final class StoreFactory {
/**
- * Single Array Key store.
- */
- private static final String SINGLE_ARRAY = "SINGLE_ARRAY";
- /**
- * Compressed single array key store.
- */
- private static final String COMPRESSED_SINGLE_ARRAY = "COMPRESSED_SINGLE_ARRAY";
- /**
* Double array data store.
*/
private static final String COMPRESSED_DOUBLE_ARRAY = "COMPRESSED_DOUBLE_ARRAY";
/**
- * key type.
- */
- private static StoreType keyType;
- /**
* value type.
*/
private static StoreType valueType;
static {
- String keytype = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.KEYSTORE_TYPE,
- CarbonCommonConstants.KEYSTORE_TYPE_DEFAULT_VAL);
String valuetype = CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.VALUESTORE_TYPE,
CarbonCommonConstants.VALUESTORE_TYPE_DEFAULT_VAL);
- // set key type
- if (COMPRESSED_SINGLE_ARRAY.equals(keytype)) {
- keyType = StoreType.COMPRESSED_SINGLE_ARRAY;
- } else if (SINGLE_ARRAY.equals(keytype)) {
- keyType = StoreType.SINGLE_ARRAY;
- } else {
- keyType = StoreType.COMPRESSED_SINGLE_ARRAY;
- }
// set value type
if (COMPRESSED_DOUBLE_ARRAY.equals(valuetype)) {
valueType = StoreType.COMPRESSED_DOUBLE_ARRAY;
@@ -89,111 +54,6 @@ public final class StoreFactory {
}
- public static NodeKeyStore createKeyStore(int size, int elementSize, boolean isLeaf,
- boolean isFileStore, long offset, String fileName, int length, FileHolder fileHolder) {
- switch (keyType) {
- case SINGLE_ARRAY:
-
- if (isFileStore) {
- return new SingleArrayKeyFileStore(size, elementSize, offset, fileName, length);
- } else {
- return new SingleArrayKeyInMemoryStore(size, elementSize, offset, fileName, fileHolder,
- length);
- }
- default:
-
- if (isLeaf) {
- if (isFileStore) {
- return new CompressedSingleArrayKeyFileStore(size, elementSize, offset, fileName,
- length);
- } else {
- return new CompressedSingleArrayKeyInMemoryStore(size, elementSize, offset, fileName,
- fileHolder, length);
- }
- } else {
- if (isFileStore) {
- return new SingleArrayKeyFileStore(size, elementSize, offset, fileName, length);
- } else {
- return new SingleArrayKeyInMemoryStore(size, elementSize, offset, fileName, fileHolder,
- length);
- }
- }
- }
- }
-
- public static NodeKeyStore createKeyStore(int size, int elementSize, boolean isLeaf) {
- switch (keyType) {
- case SINGLE_ARRAY:
-
- return new SingleArrayKeyInMemoryStore(size, elementSize);
-
- default:
-
- if (isLeaf) {
- return new CompressedSingleArrayKeyInMemoryStore(size, elementSize);
- } else {
- return new SingleArrayKeyInMemoryStore(size, elementSize);
- }
-
- }
- }
-
- public static ColumnarKeyStore createColumnarKeyStore(ColumnarKeyStoreInfo columnarKeyStoreInfo,
- FileHolder fileHolder, boolean isFileStore) {
- switch (keyType) {
- case SINGLE_ARRAY:
-
- if (isFileStore) {
- return new UnCompressedColumnarFileKeyStore(columnarKeyStoreInfo);
- } else {
- return new UnCompressedColumnarInMemoryStore(columnarKeyStoreInfo, fileHolder);
- }
- default:
-
- if (isFileStore) {
- return new CompressedColumnarFileKeyStore(columnarKeyStoreInfo);
- } else {
- return new CompressedColumnarInMemoryStore(columnarKeyStoreInfo, fileHolder);
- }
- }
- }
-
- public static NodeMeasureDataStore createDataStore(boolean isFileStore,
- ValueCompressionModel compressionModel, long[] offset, int[] length, String filePath,
- FileHolder fileHolder) {
- switch (valueType) {
-
- case COMPRESSED_DOUBLE_ARRAY:
-
- if (isFileStore) {
- return new DoubleArrayDataFileStore(compressionModel, offset, filePath, length);
- } else {
- return new DoubleArrayDataInMemoryStore(compressionModel, offset, length, filePath,
- fileHolder);
- }
-
- case HEAVY_VALUE_COMPRESSION:
-
- if (isFileStore) {
- return new HeavyCompressedDoubleArrayDataFileStore(compressionModel, offset, length,
- filePath);
- } else {
- return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel, offset, length,
- filePath, fileHolder);
- }
- default:
-
- if (isFileStore) {
- return new HeavyCompressedDoubleArrayDataFileStore(compressionModel, offset, length,
- filePath);
- } else {
- return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel, offset, length,
- filePath, fileHolder);
- }
-
- }
- }
-
public static NodeMeasureDataStore createDataStore(ValueCompressionModel compressionModel) {
switch (valueType) {
case COMPRESSED_DOUBLE_ARRAY:
@@ -210,7 +70,6 @@ public final class StoreFactory {
* enum defined.
*/
public enum StoreType {
- SINGLE_ARRAY,
COMPRESSED_SINGLE_ARRAY,
COMPRESSED_DOUBLE_ARRAY,
HEAVY_VALUE_COMPRESSION
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java b/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java
deleted file mode 100644
index cb4efc6..0000000
--- a/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.file.manager.composite;
-
-import org.carbondata.core.writer.HierarchyValueWriterForCSV;
-
-public class FileData extends FileManager {
-
- /**
- * Store Path
- */
- private String storePath;
-
- /**
- * hierarchyValueWriter
- */
- private HierarchyValueWriterForCSV hierarchyValueWriter;
-
- public FileData(String fileName, String storePath) {
- this.fileName = fileName;
- this.storePath = storePath;
- }
-
- /**
- * @return Returns the fileName.
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * @return Returns the storePath.
- */
- public String getStorePath() {
- return storePath;
- }
-
- /**
- * get Hierarchy Value writer
- *
- * @return
- */
- public HierarchyValueWriterForCSV getHierarchyValueWriter() {
- return hierarchyValueWriter;
- }
-
- /**
- * Set Hierarchy Value Writer.
- *
- * @param hierarchyValueWriter
- */
- public void setHierarchyValueWriter(HierarchyValueWriterForCSV hierarchyValueWriter) {
- this.hierarchyValueWriter = hierarchyValueWriter;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java b/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java
deleted file mode 100644
index 7e98bed..0000000
--- a/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.file.manager.composite;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-
-public class FileManager implements IFileManagerComposite {
- /**
- * listOfFileData, composite parent which holds the different objects
- */
- protected List<IFileManagerComposite> listOfFileData =
- new ArrayList<IFileManagerComposite>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- protected String fileName;
-
- @Override public void add(IFileManagerComposite customData) {
- listOfFileData.add(customData);
- }
-
- @Override public void remove(IFileManagerComposite customData) {
- listOfFileData.remove(customData);
-
- }
-
- @Override public IFileManagerComposite get(int i) {
- return listOfFileData.get(i);
- }
-
- /**
- * Renames the File/Folders
- */
- public boolean rename(IFileManagerComposite composite) {
- return false;
- }
-
- @Override public void setName(String name) {
- this.fileName = name;
- }
-
- /**
- * Return the size
- */
- public int size() {
- return listOfFileData.size();
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java b/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java
deleted file mode 100644
index a83b9f0..0000000
--- a/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.file.manager.composite;
-
-public interface IFileManagerComposite {
- /**
- * Add the data which can be either row Folder(Composite) or File
- *
- * @param customData
- */
- void add(IFileManagerComposite customData);
-
- /**
- * Remove the CustomData type object from the IFileManagerComposite object hierarchy.
- *
- * @param customData
- */
- void remove(IFileManagerComposite customData);
-
- /**
- * get the CustomData type object name
- *
- * @return CustomDataIntf type
- */
- IFileManagerComposite get(int i);
-
- /**
- * set the CustomData type object name
- *
- * @param name
- */
- void setName(String name);
-
- /**
- * Get the size
- *
- * @return
- */
- int size();
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java b/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java
deleted file mode 100644
index c651a80..0000000
--- a/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.iterator;
-
-import java.util.Iterator;
-
-/**
- * CarbonIterator adds default implement for remove. This is required for Java 7.
- * @param <E>
- */
-public abstract class CarbonIterator<E> implements Iterator<E> {
-
- @Override public abstract boolean hasNext();
-
- @Override public abstract E next();
-
- @Override public void remove() {
- throw new UnsupportedOperationException("remove");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java b/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java
deleted file mode 100644
index b77cff0..0000000
--- a/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.locks;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonProperties;
-
-/**
- * This is the abstract class of the lock implementations.This handles the
- * retrying part of the locking.
- */
-public abstract class AbstractCarbonLock implements ICarbonLock {
- private int retryCount;
-
- private int retryTimeout;
-
- public abstract boolean lock();
-
- /**
- * API for enabling the locking of file with retries.
- */
- public boolean lockWithRetries() {
- try {
- for (int i = 0; i < retryCount; i++) {
- if (lock()) {
- return true;
- } else {
- Thread.sleep(retryTimeout * 1000L);
- }
- }
- } catch (InterruptedException e) {
- return false;
- }
- return false;
- }
-
- /**
- * Initializes the retry count and retry timeout.
- * This will determine how many times to retry to acquire lock and the retry timeout.
- */
- protected void initRetry() {
- String retries = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK);
- try {
- retryCount = Integer.parseInt(retries);
- } catch (NumberFormatException e) {
- retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- String maxTimeout = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK);
- try {
- retryTimeout = Integer.parseInt(maxTimeout);
- } catch (NumberFormatException e) {
- retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
deleted file mode 100644
index 99f4ab1..0000000
--- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonProperties;
-
-/**
- * This class is a Lock factory class which is used to provide lock objects.
- * Using this lock object client can request the lock and unlock.
- */
-public class CarbonLockFactory {
-
- /**
- * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
- */
- private static String lockTypeConfigured;
-
- static {
- CarbonLockFactory.updateZooKeeperLockingStatus();
- }
-
- /**
- * This method will determine the lock type.
- *
- * @param location
- * @param lockUsage
- * @return
- */
- public static ICarbonLock getCarbonLockObj(String location, LockUsage lockUsage) {
- switch (lockTypeConfigured.toUpperCase()) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(location, lockUsage);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(location, lockUsage);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(location, lockUsage);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
- }
-
- }
-
- /**
- * This method will set the zookeeper status whether zookeeper to be used for locking or not.
- */
- private static void updateZooKeeperLockingStatus() {
- lockTypeConfigured = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
deleted file mode 100644
index 745dfa1..0000000
--- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-
-/**
- * This class is used to handle the HDFS File locking.
- * This is acheived using the concept of acquiring the data out stream using Append option.
- */
-public class HdfsFileLock extends AbstractCarbonLock {
-
- /**
- * location hdfs file location
- */
- private String location;
-
- /**
- * lockUsage is used to determine the type of the lock. according to this the lock
- * folder will change.
- */
- private LockUsage lockUsage;
-
- private DataOutputStream dataOutputStream;
-
- /**
- * @param location
- * @param lockUsage
- */
- public HdfsFileLock(String location, LockUsage lockUsage) {
- this.location = location;
- this.lockUsage = lockUsage;
- this.location = location + File.separator + this.lockUsage;
- initRetry();
- }
-
- /* (non-Javadoc)
- * @see org.carbondata.core.locks.ICarbonLock#lock()
- */
- @Override public boolean lock() {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
- }
- dataOutputStream =
- FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
-
- return true;
-
- } catch (IOException e) {
- return false;
- }
- }
-
- /* (non-Javadoc)
- * @see org.carbondata.core.locks.ICarbonLock#unlock()
- */
- @Override public boolean unlock() {
- if (null != dataOutputStream) {
- try {
- dataOutputStream.close();
- } catch (IOException e) {
- return false;
- }
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java b/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java
deleted file mode 100644
index af97c2e..0000000
--- a/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-/**
- * Carbon Lock Interface which handles the locking and unlocking.
- */
-public interface ICarbonLock {
-
- /**
- * Does the unlocking of the acquired lock.
- *
- * @return
- */
- boolean unlock();
-
- /**
- * This will acquire the lock and if it doesnt get then it will retry after the confiured time.
- *
- * @return
- */
- boolean lockWithRetries();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
deleted file mode 100644
index 362b86f..0000000
--- a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-
-/**
- * This class handles the file locking in the local file system.
- * This will be handled using the file channel lock API.
- */
-public class LocalFileLock extends AbstractCarbonLock {
- /**
- * location is the location of the lock file.
- */
- private String location;
-
- /**
- * lockUsage will determine the lock folder. so that similar locks will try to acquire
- * same lock file.
- */
- private LockUsage lockUsage;
-
- /**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
- * channel is the FileChannel of the lock file.
- */
- private FileChannel channel;
-
- /**
- * fileLock NIO FileLock Object
- */
- private FileLock fileLock;
-
- public static final String tmpPath;
-
- private String cubeName;
-
- private String schemaName;
-
- /**
- * LOGGER for logging the messages.
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LocalFileLock.class.getName());
-
- static {
- tmpPath = System.getProperty("java.io.tmpdir");
- }
-
- /**
- * @param location
- * @param lockUsage
- */
- public LocalFileLock(String location, LockUsage lockUsage) {
- this.lockUsage = lockUsage;
- location = location.replace("\\", "/");
- String tempStr = location.substring(0, location.lastIndexOf('/'));
- schemaName = tempStr.substring(tempStr.lastIndexOf('/') + 1, tempStr.length());
-
- cubeName = location.substring(location.lastIndexOf('/') + 1, location.length());
- this.location =
- tmpPath + File.separator + schemaName + File.separator + cubeName + File.separator
- + this.lockUsage;
- initRetry();
- }
-
- /**
- * Lock API for locking of the file channel of the lock file.
- *
- * @return
- */
- @Override public boolean lock() {
- try {
- String schemaFolderPath = tmpPath + File.separator + schemaName;
- String cubeFolderPath = schemaFolderPath + File.separator + cubeName;
- // create dir with schema name in tmp location.
- if (!FileFactory.isFileExist(schemaFolderPath, FileFactory.getFileType(tmpPath))) {
- FileFactory.mkdirs(schemaFolderPath, FileFactory.getFileType(tmpPath));
- }
-
- // create dir with cube name in tmp location.
- if (!FileFactory.isFileExist(cubeFolderPath, FileFactory.getFileType(tmpPath))) {
- FileFactory.mkdirs(cubeFolderPath, FileFactory.getFileType(tmpPath));
- }
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
- }
-
- fileOutputStream = new FileOutputStream(location);
- channel = fileOutputStream.getChannel();
- try {
- fileLock = channel.tryLock();
- } catch (OverlappingFileLockException e) {
- return false;
- }
- if (null != fileLock) {
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- return false;
- }
-
- }
-
- /**
- * Unlock API for unlocking of the acquired lock.
- *
- * @return
- */
- @Override public boolean unlock() {
- boolean status;
- try {
- if (null != fileLock) {
- fileLock.release();
- }
- status = true;
- } catch (IOException e) {
- status = false;
- } finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
- }
- }
- return status;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/LockUsage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/LockUsage.java b/core/src/main/java/org/carbondata/core/locks/LockUsage.java
deleted file mode 100644
index 850b6bf..0000000
--- a/core/src/main/java/org/carbondata/core/locks/LockUsage.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-/**
- * This enum is used to define the usecase of the lock.
- * Each enum value is one specific lock case.
- */
-public enum LockUsage {
- METADATA_LOCK,
- COMPACTION_LOCK;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
deleted file mode 100644
index 8b04ac1..0000000
--- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.locks;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.constants.CarbonCommonConstants;
-
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * For Handling the zookeeper locking implementation
- */
-public class ZooKeeperLocking extends AbstractCarbonLock {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
-
- /**
- * zk is the zookeeper client instance
- */
- private static ZooKeeper zk;
-
- /**
- * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
- * maintained.
- */
- private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
-
- /**
- * Unique folder for each cube with SchemaName_CubeName
- */
- private final String tableIdFolder;
-
- /**
- * lockName is the name of the lock to use. This name should be same for every process that want
- * to share the same lock
- */
- private String lockName;
-
- /**
- * lockPath is the unique path created for the each instance of the carbon lock.
- */
- private String lockPath;
-
- private String lockTypeFolder;
-
- /**
- * @param lockUsage
- */
- public ZooKeeperLocking(String location, LockUsage lockUsage) {
- this.lockName = CarbonCommonConstants.ZOOKEEPER_LOCK;
- this.lockTypeFolder = zooKeeperLocation;
- location = location.replace("\\", "/");
- String tempStr = location.substring(0, location.lastIndexOf('/'));
- String schemaName = tempStr.substring(tempStr.lastIndexOf('/') + 1, tempStr.length());
-
- String cubeName = location.substring(location.lastIndexOf('/') + 1, location.length());
-
- this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + schemaName
- + '.' + cubeName;
-
- zk = ZookeeperInit.getInstance().getZookeeper();
-
- this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + schemaName
- + '.' + cubeName + CarbonCommonConstants.FILE_SEPARATOR
- + lockUsage.toString();
- try {
- createBaseNode();
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.tableIdFolder, true)) {
- zk.create(this.tableIdFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.lockTypeFolder, true)) {
- zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- }
- initRetry();
- }
-
- /**
- * Creating a znode in which all the znodes (lock files )are maintained.
- */
- private void createBaseNode() throws KeeperException, InterruptedException {
- if (null == zk.exists(zooKeeperLocation, true)) {
- // creating a znode in which all the znodes (lock files )are maintained.
- zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- }
-
- /**
- * Handling of the locking mechanism using zoo keeper.
- */
- @Override public boolean lock() {
- try {
- // create the lock file with lockName.
- lockPath =
- zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
- Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
-
- // get the children present in zooKeeperLocation.
- List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
-
- // sort the childrens
- Collections.sort(nodes);
-
- // here the logic is , for each lock request zookeeper will create a file ending with
- // incremental digits.
- // so first request will be 00001 next is 00002 and so on.
- // if the current request is 00002 and already one previous request(00001) is present then get
- // children will give both nodes.
- // after the sort we are checking if the lock path is first or not .if it is first then lock
- // has been acquired.
-
- if (lockPath.endsWith(nodes.get(0))) {
- return true;
- } else {
- // if locking failed then deleting the created lock as next time again new lock file will be
- // created.
- zk.delete(lockPath, -1);
- return false;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- }
-
- /**
- * @return status where lock file is unlocked or not.
- */
- @Override public boolean unlock() {
- try {
- // exists will return null if the path doesn't exists.
- if (null != zk.exists(lockPath, true)) {
- zk.delete(lockPath, -1);
- lockPath = null;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java b/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java
deleted file mode 100644
index ba2ad57..0000000
--- a/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.carbondata.core.locks;
-
-import java.io.IOException;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * This is a singleton class for initialization of zookeeper client.
- */
-public class ZookeeperInit {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZookeeperInit.class.getName());
-
- private static ZookeeperInit zooKeeperInit;
- /**
- * zk is the zookeeper client instance
- */
- private ZooKeeper zk;
-
- private ZookeeperInit(String zooKeeperUrl) {
-
- int sessionTimeOut = 100000;
- try {
- zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher());
-
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
-
- }
-
- public static ZookeeperInit getInstance(String zooKeeperUrl) {
-
- if (null == zooKeeperInit) {
- synchronized (ZookeeperInit.class) {
- if (null == zooKeeperInit) {
- LOGGER.info("Initiating Zookeeper client.");
- zooKeeperInit = new ZookeeperInit(zooKeeperUrl);
- }
- }
- }
- return zooKeeperInit;
-
- }
-
- public static ZookeeperInit getInstance() {
- return zooKeeperInit;
- }
-
- public ZooKeeper getZookeeper() {
- return zk;
- }
-
- private static class DummyWatcher implements Watcher {
- public void process(WatchedEvent event) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java b/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java
deleted file mode 100644
index d1cf630..0000000
--- a/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.metadata;
-
-public class LevelNormalizedProps {
- /**
- * isLevelNormalized
- */
- private boolean isLevelNormalized;
-
- /**
- * is this level present in fact file
- */
- private boolean isDimInFact;
-
- /**
- * hierarchy Name
- */
- private String hierName;
-
- /**
- * dimension Name
- */
- private String dimName;
-
- /**
- *
- */
- private boolean hasAll;
-
- /**
- * Constructor
- */
- public LevelNormalizedProps() {
-
- }
-
- /**
- * @return
- */
- public String getDimName() {
- return dimName;
- }
-
- /**
- * @param dimName
- */
- public void setDimName(String dimName) {
- this.dimName = dimName;
- }
-
- /**
- * @return
- */
- public String getHierName() {
- return hierName;
- }
-
- /**
- * @param hierName
- */
- public void setHierName(String hierName) {
- this.hierName = hierName;
- }
-
- public boolean isLevelNormalized() {
- return isLevelNormalized;
- }
-
- public void setLevelNormalized(boolean isLevelNormalized) {
- this.isLevelNormalized = isLevelNormalized;
- }
-
- public boolean isDimInFact() {
- return isDimInFact;
- }
-
- public void setDimInFact(boolean isDimInFact) {
- this.isDimInFact = isDimInFact;
- }
-
- public boolean isHasAll() {
- return hasAll;
- }
-
- public void setHasAll(boolean hasAll) {
- this.hasAll = hasAll;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java b/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java
deleted file mode 100644
index c0dcfc4..0000000
--- a/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.metadata;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashSet;
-
-import org.carbondata.core.keygenerator.KeyGenerator;
-
-public class SliceMetaData implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = 3046237866264840878L;
-
- /**
- * Array of dimensions declared.
- */
- private String[] dimensions;
-
- private String complexTypeString;
- /**
- * actualDimensions
- */
- private String[] actualDimensions;
- /**
- * Array of measures declared.
- */
- private String[] measures;
- /**
- * measuresAggregator
- */
- private String[] measuresAggregator;
- /**
- * Array of newDimensions declared.
- */
- private String[] newDimensions;
- /**
- * actualNewDimensions
- */
- private String[] newActualDimensions;
- /**
- * tableNamesToLoadMandatory
- */
- private HashSet<String> tableNamesToLoadMandatory;
- /**
- * Array of newMeasures declared.
- */
- private String[] newMeasures;
- /**
- * Array of newMsrDfts declared.
- */
- private double[] newMsrDfts;
- /**
- * newMeasuresAggregator
- */
- private String[] newMeasuresAggregator;
- /**
- * heirAnKeySize
- */
- private String heirAnKeySize;
- /**
- * KeyGenerator declared.
- */
- private KeyGenerator keyGenerator;
- /**
- * dimLens
- */
- private int[] dimLens;
- /**
- * actualDimLens
- */
- private int[] actualDimLens;
- /**
- * newDimLens
- */
- private int[] newDimLens;
- /**
- * newActualDimLens
- */
- private int[] newActualDimLens;
- /**
- * oldDimsNewCardinality
- */
- private int[] oldDimsNewCardinality;
- /**
- * isDimCarinalityChanged
- */
- private boolean isDimCarinalityChanged;
- private String[] newDimsDefVals;
- private int[] newDimsSurrogateKeys;
- /**
- * new keygenerator
- */
- private KeyGenerator newKeyGenerator;
-
- public String getComplexTypeString() {
- return complexTypeString;
- }
-
- public void setComplexTypeString(String complexTypeString) {
- this.complexTypeString = complexTypeString;
- }
-
- public int[] getNewDimLens() {
- return newDimLens;
- }
-
- public void setNewDimLens(int[] newDimLens) {
- this.newDimLens = newDimLens;
- }
-
- public int[] getDimLens() {
- return dimLens;
- }
-
- public void setDimLens(int[] dimLens) {
- this.dimLens = dimLens;
- }
-
- public KeyGenerator getNewKeyGenerator() {
- return newKeyGenerator;
- }
-
- public void setNewKeyGenerator(KeyGenerator newKeyGenerator) {
- this.newKeyGenerator = newKeyGenerator;
- }
-
- public String[] getDimensions() {
- return dimensions;
- }
-
- public void setDimensions(String[] dimensions) {
- this.dimensions = dimensions;
- }
-
- public String[] getMeasures() {
- return measures;
- }
-
- public void setMeasures(String[] measures) {
- this.measures = measures;
- }
-
- public String[] getNewDimensions() {
- return newDimensions;
- }
-
- public void setNewDimensions(String[] newDimensions) {
- this.newDimensions = newDimensions;
- }
-
- public String[] getNewMeasures() {
- return newMeasures;
- }
-
- public void setNewMeasures(String[] newMeasures) {
- this.newMeasures = newMeasures;
- }
-
- public double[] getNewMsrDfts() {
- return newMsrDfts;
- }
-
- public void setNewMsrDfts(double[] newMsrDfts) {
- this.newMsrDfts = newMsrDfts;
- }
-
- public KeyGenerator getKeyGenerator() {
- return keyGenerator;
- }
-
- public void setKeyGenerator(KeyGenerator keyGenerator) {
- this.keyGenerator = keyGenerator;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(dimensions);
- result = prime * result + ((keyGenerator == null) ? 0 : keyGenerator.hashCode());
- result = prime * result + Arrays.hashCode(measures);
- result = prime * result + Arrays.hashCode(newDimensions);
- result = prime * result + Arrays.hashCode(newMeasures);
- result = prime * result + Arrays.hashCode(newMsrDfts);
- result = prime * result + Arrays.hashCode(dimLens);
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (obj instanceof SliceMetaData) {
- SliceMetaData other = (SliceMetaData) obj;
- if (Arrays.equals(dimensions, other.dimensions) && Arrays
- .equals(measuresAggregator, other.measuresAggregator) && Arrays
- .equals(dimLens, other.dimLens)) {
- return true;
- }
- }
- return false;
- }
-
- public boolean isSameAs(SliceMetaData other) {
- return (Arrays.equals(newDimensions, other.newDimensions) && Arrays
- .equals(newMeasures, other.newMeasures));
- }
-
- public String[] getMeasuresAggregator() {
- return measuresAggregator;
- }
-
- public void setMeasuresAggregator(String[] measuresAggregator) {
- this.measuresAggregator = measuresAggregator;
- }
-
- public String[] getNewMeasuresAggregator() {
- return newMeasuresAggregator;
- }
-
- public void setNewMeasuresAggregator(String[] newMeasuresAggregator) {
- this.newMeasuresAggregator = newMeasuresAggregator;
- }
-
- public String getHeirAnKeySize() {
- return heirAnKeySize;
- }
-
- public void setHeirAnKeySize(String heirAnKeySize) {
- this.heirAnKeySize = heirAnKeySize;
- }
-
- public boolean isDimCarinalityChanged() {
- return isDimCarinalityChanged;
- }
-
- public void setDimCarinalityChanged(boolean isDimCarinalityChanged) {
- this.isDimCarinalityChanged = isDimCarinalityChanged;
- }
-
- public int[] getOldDimsNewCardinality() {
- return oldDimsNewCardinality;
- }
-
- public void setOldDimsNewCardinality(int[] oldDimsNewCardinality) {
- this.oldDimsNewCardinality = oldDimsNewCardinality;
- }
-
- public String[] getNewActualDimensions() {
- return newActualDimensions;
- }
-
- public void setNewActualDimensions(String[] newActualDimensions) {
- this.newActualDimensions = newActualDimensions;
- }
-
- public int[] getNewActualDimLens() {
- return newActualDimLens;
- }
-
- public void setNewActualDimLens(int[] newActualDimLens) {
- this.newActualDimLens = newActualDimLens;
- }
-
- public String[] getActualDimensions() {
- return actualDimensions;
- }
-
- public void setActualDimensions(String[] actualDimensions) {
- this.actualDimensions = actualDimensions;
- }
-
- public int[] getActualDimLens() {
- return actualDimLens;
- }
-
- public void setActualDimLens(int[] actualDimLens) {
- this.actualDimLens = actualDimLens;
- }
-
- public HashSet<String> getTableNamesToLoadMandatory() {
- return tableNamesToLoadMandatory;
- }
-
- public void setTableNamesToLoadMandatory(HashSet<String> tableNamesToLoadMandatory) {
- this.tableNamesToLoadMandatory = tableNamesToLoadMandatory;
- }
-
- /**
- * return the new dimensions default values
- */
- public String[] getNewDimsDefVals() {
- return newDimsDefVals;
- }
-
- /**
- * set the default values of new dimensions added
- */
- public void setNewDimsDefVals(String[] newDimsDefVals) {
- this.newDimsDefVals = newDimsDefVals;
- }
-
- /**
- * return the surrogate keys of new dimension values
- */
- public int[] getNewDimsSurrogateKeys() {
- return newDimsSurrogateKeys;
- }
-
- /**
- * set the surrogate keys for the new dimension values
- */
- public void setNewDimsSurrogateKeys(int[] newDimsSurrogateKeys) {
- this.newDimsSurrogateKeys = newDimsSurrogateKeys;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
index a359339..e2cdf64 100644
--- a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
@@ -19,19 +19,9 @@
package org.carbondata.core.util;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-import org.carbondata.core.datastorage.store.impl.FileFactory.FileType;
-
-import org.pentaho.di.core.exception.KettleException;
/**
* Util class for merge activities of 2 loads.
@@ -44,71 +34,6 @@ public class CarbonMergerUtil {
private static final LogService LOGGER =
LogServiceFactory.getLogService(CarbonMergerUtil.class.getName());
- public static List<CarbonSliceAndFiles> getSliceAndFilesList(String storeLocation,
- String tableName, FileType fileType, List<String> loadsToBeMerged) {
- try {
- if (!FileFactory.isFileExist(storeLocation, fileType)) {
- return new ArrayList<CarbonSliceAndFiles>(0);
- }
- } catch (IOException e) {
- LOGGER.error("Error occurred :: " + e.getMessage());
- }
- CarbonFile file = FileFactory.getCarbonFile(storeLocation, fileType);
-
- CarbonFile[] listFiles = CarbonUtil.listFiles(file);
- if (null == listFiles || listFiles.length < 0) {
- return new ArrayList<CarbonSliceAndFiles>(0);
- }
- Arrays.sort(listFiles, new CarbonFileFolderComparator());
- listFiles = getMergeFilesList(loadsToBeMerged, listFiles);
-
- return CarbonUtil.getSliceAndFilesList(tableName, listFiles, fileType);
- }
-
- private static CarbonFile[] getMergeFilesList(List<String> loadsToBeMerged,
- CarbonFile[] listFiles) {
- CarbonFile[] carbonFile = new CarbonFile[loadsToBeMerged.size()];
- int i = 0;
- for (CarbonFile listFile : listFiles) {
- String loadName = listFile.getName();
- for (String load : loadsToBeMerged) {
- if ((CarbonCommonConstants.LOAD_FOLDER + load).equalsIgnoreCase(loadName)) {
- carbonFile[i++] = listFile;
- }
- }
- }
- return carbonFile;
- }
-
- public static int[] mergeLevelMetadata(String[] sliceLocation, String tableName,
- String destinationLocation) {
- int[][] cardinalityOfLoads = new int[sliceLocation.length][];
- int i = 0;
- for (String loadFolderLoacation : sliceLocation) {
- try {
- cardinalityOfLoads[i++] = CarbonUtil.getCardinalityFromLevelMetadataFile(
- loadFolderLoacation + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName
- + ".metadata");
- } catch (CarbonUtilException e) {
- LOGGER
- .error("Error occurred :: " + e.getMessage());
- }
- }
- int[] MaxCardinality = new int[cardinalityOfLoads[0].length];
-
- for (int k = 0; k < cardinalityOfLoads[0].length; k++) {
- MaxCardinality[k] = Math.max(cardinalityOfLoads[0][k], cardinalityOfLoads[1][k]);
- }
-
- try {
- CarbonUtil.writeLevelCardinalityFile(destinationLocation, tableName, MaxCardinality);
- } catch (KettleException e) {
- LOGGER.error("Error occurred :: " + e.getMessage());
- }
-
- return MaxCardinality;
- }
-
public static int[] getCardinalityFromLevelMetadata(String path, String tableName) {
int[] localCardinality = null;
try {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java b/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java
deleted file mode 100644
index 474f247..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.util;
-
-import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.carbondata.core.keygenerator.KeyGenerator;
-
-public class CarbonSliceAndFiles {
- /**
- * slice path
- */
- private String path;
-
- /**
- * slice fact files
- */
- private CarbonFile[] sliceFactFilesList;
-
- private KeyGenerator keyGen;
-
- /**
- * This method will be used get the slice fact files
- *
- * @return slice fact files
- */
- public CarbonFile[] getSliceFactFilesList() {
- return sliceFactFilesList;
- }
-
- /**
- * This method will be used to set the slice fact files
- *
- * @param sliceFactFilesList
- */
- public void setSliceFactFilesList(CarbonFile[] sliceFactFilesList) {
- this.sliceFactFilesList = sliceFactFilesList;
- }
-
- /**
- * This method will return the slice path
- *
- * @return slice path
- */
- public String getPath() {
- return path;
- }
-
- /**
- * This method will be used to set the slice path
- *
- * @param path
- */
- public void setPath(String path) {
- this.path = path;
- }
-
- /**
- * @return the keyGen
- */
- public KeyGenerator getKeyGen() {
- return keyGen;
- }
-
- /**
- * @param keyGen the keyGen to set
- */
- public void setKeyGen(KeyGenerator keyGen) {
- this.keyGen = keyGen;
- }
-
-}