You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:09:08 UTC
[23/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/AbsoluteTableIdentifier.java b/core/src/main/java/org/carbondata/core/carbon/AbsoluteTableIdentifier.java
deleted file mode 100644
index 7234a82..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/AbsoluteTableIdentifier.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon;
-
-import java.io.Serializable;
-
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-
-/**
- * identifier which will have store path and carbon table identifier
- */
-public class AbsoluteTableIdentifier implements Serializable {
-
- /**
- * serializable version
- */
- private static final long serialVersionUID = 4695047103484427506L;
-
- /**
- * path of the store
- */
- private String storePath;
-
- /**
- * carbon table identifier which will have table name and table database
- * name
- */
- private CarbonTableIdentifier carbonTableIdentifier;
-
- public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
- //TODO this should be moved to common place where path handling will be handled
- this.storePath = FileFactory.getUpdatedFilePath(storePath);
- this.carbonTableIdentifier = carbonTableIdentifier;
- }
-
- /**
- * @return the storePath
- */
- public String getStorePath() {
- return storePath;
- }
-
- /**
- * @return the carbonTableIdentifier
- */
- public CarbonTableIdentifier getCarbonTableIdentifier() {
- return carbonTableIdentifier;
- }
-
- /**
- * to get the hash code
- */
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result =
- prime * result + ((carbonTableIdentifier == null) ? 0 : carbonTableIdentifier.hashCode());
- result = prime * result + ((storePath == null) ? 0 : storePath.hashCode());
- return result;
- }
-
- /**
- * to check this class is equal to
- * other object passed
- *
- * @param obj other object
- */
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof AbsoluteTableIdentifier)) {
- return false;
- }
- AbsoluteTableIdentifier other = (AbsoluteTableIdentifier) obj;
- if (carbonTableIdentifier == null) {
- if (other.carbonTableIdentifier != null) {
- return false;
- }
- } else if (!carbonTableIdentifier.equals(other.carbonTableIdentifier)) {
- return false;
- }
- if (storePath == null) {
- if (other.storePath != null) {
- return false;
- }
- } else if (!storePath.equals(other.storePath)) {
- return false;
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/CarbonDataLoadSchema.java b/core/src/main/java/org/carbondata/core/carbon/CarbonDataLoadSchema.java
deleted file mode 100644
index f23f600..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/CarbonDataLoadSchema.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-
-/**
- * Wrapper Data Load Schema object which will be used to
- * support relation while data loading
- */
-public class CarbonDataLoadSchema implements Serializable {
-
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * CarbonTable info
- */
- private CarbonTable carbonTable;
-
- /**
- * dimension table and relation info
- */
- private List<DimensionRelation> dimensionRelationList;
-
- /**
- * CarbonDataLoadSchema constructor which takes CarbonTable
- *
- * @param carbonTable
- */
- public CarbonDataLoadSchema(CarbonTable carbonTable) {
- this.carbonTable = carbonTable;
- this.dimensionRelationList = new ArrayList<DimensionRelation>();
- }
-
- /**
- * get dimension relation list
- *
- * @return dimensionRelationList
- */
- public List<DimensionRelation> getDimensionRelationList() {
- return dimensionRelationList;
- }
-
- /**
- * set dimensionrelation list
- *
- * @param dimensionRelationList
- */
- public void setDimensionRelationList(List<DimensionRelation> dimensionRelationList) {
- this.dimensionRelationList = dimensionRelationList;
- }
-
- /**
- * get carbontable
- *
- * @return carbonTable
- */
- public CarbonTable getCarbonTable() {
- return carbonTable;
- }
-
- /**
- * Dimension Relation object which will be filled from
- * Load DML Command to support normalized table data load
- */
- public static class DimensionRelation implements Serializable {
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * dimension tableName
- */
- private String tableName;
-
- /**
- * dimensionSource csv path
- */
- private String dimensionSource;
-
- /**
- * relation with fact and dimension table
- */
- private Relation relation;
-
- /**
- * Columns to selected from dimension table.
- * Hierarchy in-memory table should be prepared
- * based on selected columns
- */
- private List<String> columns;
-
- /**
- * constructor
- *
- * @param tableName - dimension table name
- * @param dimensionSource - source file path
- * @param relation - fact foreign key with dimension primary key mapping
- * @param columns - list of columns to be used from this dimension table
- */
- public DimensionRelation(String tableName, String dimensionSource, Relation relation,
- List<String> columns) {
- this.tableName = tableName;
- this.dimensionSource = dimensionSource;
- this.relation = relation;
- this.columns = columns;
- }
-
- /**
- * @return tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- /**
- * @return dimensionSource
- */
- public String getDimensionSource() {
- return dimensionSource;
- }
-
- /**
- * @return relation
- */
- public Relation getRelation() {
- return relation;
- }
-
- /**
- * @return columns
- */
- public List<String> getColumns() {
- return columns;
- }
- }
-
- /**
- * Relation class to specify fact foreignkey column with
- * dimension primary key column
- */
- public static class Relation implements Serializable {
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * Fact foreign key column
- */
- private String factForeignKeyColumn;
-
- /**
- * dimension primary key column
- */
- private String dimensionPrimaryKeyColumn;
-
- /**
- * constructor
- *
- * @param factForeignKeyColumn - Fact Table Foreign key
- * @param dimensionPrimaryKeyColumn - Dimension Table primary key
- */
- public Relation(String factForeignKeyColumn, String dimensionPrimaryKeyColumn) {
- this.factForeignKeyColumn = factForeignKeyColumn;
- this.dimensionPrimaryKeyColumn = dimensionPrimaryKeyColumn;
- }
-
- /**
- * @return factForeignKeyColumn
- */
- public String getFactForeignKeyColumn() {
- return factForeignKeyColumn;
- }
-
- /**
- * @return dimensionPrimaryKeyColumn
- */
- public String getDimensionPrimaryKeyColumn() {
- return dimensionPrimaryKeyColumn;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/CarbonTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/CarbonTableIdentifier.java b/core/src/main/java/org/carbondata/core/carbon/CarbonTableIdentifier.java
deleted file mode 100644
index 949522c..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/CarbonTableIdentifier.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.carbon;
-
-import java.io.Serializable;
-
-/**
- * Identifier class which will hold the table qualified name
- */
-public class CarbonTableIdentifier implements Serializable {
-
- /**
- * database name
- */
- private String databaseName;
-
- /**
- * table name
- */
- private String tableName;
-
- /**
- * table id
- */
- private String tableId;
-
- /**
- * constructor
- */
- public CarbonTableIdentifier(String databaseName, String tableName, String tableId) {
- this.databaseName = databaseName;
- this.tableName = tableName;
- this.tableId = tableId;
- }
-
- /**
- * return database name
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * return table name
- */
- public String getTableName() {
- return tableName;
- }
-
- /**
- * @return tableId
- */
- public String getTableId() {
- return tableId;
- }
-
- /**
- * @return table unique name
- */
- public String getTableUniqueName() {
- return databaseName + '_' + tableName;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((databaseName == null) ? 0 : databaseName.hashCode());
- result = prime * result + ((tableId == null) ? 0 : tableId.hashCode());
- result = prime * result + ((tableName == null) ? 0 : tableName.hashCode());
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- CarbonTableIdentifier other = (CarbonTableIdentifier) obj;
- if (databaseName == null) {
- if (other.databaseName != null) {
- return false;
- }
- } else if (!databaseName.equals(other.databaseName)) {
- return false;
- }
- if (tableId == null) {
- if (other.tableId != null) {
- return false;
- }
- } else if (!tableId.equals(other.tableId)) {
- return false;
- }
- if (tableName == null) {
- if (other.tableName != null) {
- return false;
- }
- } else if (!tableName.equals(other.tableName)) {
- return false;
- }
- return true;
- }
-
- /*
- * @return table unidque name
- */
- @Override public String toString() {
- return databaseName + '_' + tableName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java b/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
deleted file mode 100644
index 74d8b4d..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/ColumnIdentifier.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon;
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.carbondata.core.carbon.metadata.datatype.DataType;
-
-/**
- * Column unique identifier
- */
-public class ColumnIdentifier implements Serializable {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * column id
- */
- private String columnId;
-
- /**
- * column properties
- */
- private Map<String, String> columnProperties;
-
- private DataType dataType;
-
- /**
- * @param columnId
- * @param columnProperties
- */
- public ColumnIdentifier(String columnId, Map<String, String> columnProperties,
- DataType dataType) {
- this.columnId = columnId;
- this.columnProperties = columnProperties;
- this.dataType = dataType;
- }
-
- /**
- * @return columnId
- */
- public String getColumnId() {
- return columnId;
- }
-
- /**
- * @param columnProperty
- * @return
- */
- public String getColumnProperty(String columnProperty) {
- if (null != columnProperties) {
- return columnProperties.get(columnProperty);
- }
- return null;
- }
-
- public DataType getDataType() {
- return this.dataType;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((columnId == null) ? 0 : columnId.hashCode());
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- ColumnIdentifier other = (ColumnIdentifier) obj;
- if (columnId == null) {
- if (other.columnId != null) {
- return false;
- }
- } else if (!columnId.equals(other.columnId)) {
- return false;
- }
- return true;
- }
-
- @Override public String toString() {
- return "ColumnIdentifier [columnId=" + columnId + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/BTreeBuilderInfo.java b/core/src/main/java/org/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
deleted file mode 100644
index 338d80e..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-import java.util.List;
-
-import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-
-/**
- * below class holds the meta data requires to build the blocks
- */
-public class BTreeBuilderInfo {
-
- /**
- * holds all the information about data
- * file meta data
- */
- private List<DataFileFooter> footerList;
-
- /**
- * size of the each column value size
- * this will be useful for reading
- */
- private int[] dimensionColumnValueSize;
-
- public BTreeBuilderInfo(List<DataFileFooter> footerList,
- int[] dimensionColumnValueSize) {
- this.dimensionColumnValueSize = dimensionColumnValueSize;
- this.footerList = footerList;
- }
-
- /**
- * @return the eachDimensionBlockSize
- */
- public int[] getDimensionColumnValueSize() {
- return dimensionColumnValueSize;
- }
-
- /**
- * @return the footerList
- */
- public List<DataFileFooter> getFooterList() {
- return footerList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/BlockIndexStore.java b/core/src/main/java/org/carbondata/core/carbon/datastore/BlockIndexStore.java
deleted file mode 100644
index 7ef7a24..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/BlockIndexStore.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.carbondata.core.carbon.datastore.block.AbstractIndex;
-import org.carbondata.core.carbon.datastore.block.BlockIndex;
-import org.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.carbondata.core.carbon.datastore.exception.IndexBuilderException;
-import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonProperties;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.util.CarbonUtilException;
-
-/**
- * Singleton Class to handle loading, unloading,clearing,storing of the table
- * blocks
- */
-public class BlockIndexStore {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockIndexStore.class.getName());
- /**
- * singleton instance
- */
- private static final BlockIndexStore CARBONTABLEBLOCKSINSTANCE = new BlockIndexStore();
-
- /**
- * map to hold the table and its list of blocks
- */
- private Map<AbsoluteTableIdentifier, Map<TableBlockInfo, AbstractIndex>> tableBlocksMap;
-
- /**
- * map of block info to lock object map, while loading the btree this will be filled
- * and removed after loading the tree for that particular block info, this will be useful
- * while loading the tree concurrently so only block level lock will be applied another
- * block can be loaded concurrently
- */
- private Map<TableBlockInfo, Object> blockInfoLock;
-
- /**
- * table and its lock object to this will be useful in case of concurrent
- * query scenario when more than one query comes for same table and in that
- * case it will ensure that only one query will able to load the blocks
- */
- private Map<AbsoluteTableIdentifier, Object> tableLockMap;
-
- private BlockIndexStore() {
- tableBlocksMap =
- new ConcurrentHashMap<AbsoluteTableIdentifier, Map<TableBlockInfo, AbstractIndex>>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- tableLockMap = new ConcurrentHashMap<AbsoluteTableIdentifier, Object>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- blockInfoLock = new ConcurrentHashMap<TableBlockInfo, Object>();
- }
-
- /**
- * Return the instance of this class
- *
- * @return singleton instance
- */
- public static BlockIndexStore getInstance() {
- return CARBONTABLEBLOCKSINSTANCE;
- }
-
- /**
- * below method will be used to load the block which are not loaded and to
- * get the loaded blocks if all the blocks which are passed is loaded then
- * it will not load , else it will load.
- *
- * @param tableBlocksInfos list of blocks to be loaded
- * @param absoluteTableIdentifier absolute Table Identifier to identify the table
- * @throws IndexBuilderException
- */
- public List<AbstractIndex> loadAndGetBlocks(List<TableBlockInfo> tableBlocksInfos,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IndexBuilderException {
- AbstractIndex[] loadedBlock = new AbstractIndex[tableBlocksInfos.size()];
- addTableLockObject(absoluteTableIdentifier);
- // sort the block info
- // so block will be loaded in sorted order this will be required for
- // query execution
- Collections.sort(tableBlocksInfos);
- // get the instance
- Object lockObject = tableLockMap.get(absoluteTableIdentifier);
- Map<TableBlockInfo, AbstractIndex> tableBlockMapTemp = null;
- int numberOfCores = 1;
- try {
- numberOfCores = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUM_CORES,
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
- } catch (NumberFormatException e) {
- numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- }
- ExecutorService executor = Executors.newFixedThreadPool(numberOfCores);
- // Acquire the lock to ensure only one query is loading the table blocks
- // if same block is assigned to both the queries
- synchronized (lockObject) {
- tableBlockMapTemp = tableBlocksMap.get(absoluteTableIdentifier);
- // if it is loading for first time
- if (null == tableBlockMapTemp) {
- tableBlockMapTemp = new ConcurrentHashMap<TableBlockInfo, AbstractIndex>();
- tableBlocksMap.put(absoluteTableIdentifier, tableBlockMapTemp);
- }
- }
- AbstractIndex tableBlock = null;
- List<Future<AbstractIndex>> blocksList = new ArrayList<Future<AbstractIndex>>();
- int counter = -1;
- for (TableBlockInfo blockInfo : tableBlocksInfos) {
- counter++;
- // if table block is already loaded then do not load
- // that block
- tableBlock = tableBlockMapTemp.get(blockInfo);
- // if block is not loaded
- if (null == tableBlock) {
- // check any lock object is present in
- // block info lock map
- Object blockInfoLockObject = blockInfoLock.get(blockInfo);
- // if lock object is not present then acquire
- // the lock in block info lock and add a lock object in the map for
- // particular block info, added double checking mechanism to add the lock
- // object so in case of concurrent query we for same block info only one lock
- // object will be added
- if (null == blockInfoLockObject) {
- synchronized (blockInfoLock) {
- // again checking the block info lock, to check whether lock object is present
- // or not if now also not present then add a lock object
- blockInfoLockObject = blockInfoLock.get(blockInfo);
- if (null == blockInfoLockObject) {
- blockInfoLockObject = new Object();
- blockInfoLock.put(blockInfo, blockInfoLockObject);
- }
- }
- }
- //acquire the lock for particular block info
- synchronized (blockInfoLockObject) {
- // check again whether block is present or not to avoid the
- // same block is loaded
- //more than once in case of concurrent query
- tableBlock = tableBlockMapTemp.get(blockInfo);
- // if still block is not present then load the block
- if (null == tableBlock) {
- blocksList.add(executor.submit(new BlockLoaderThread(blockInfo, tableBlockMapTemp)));
- }
- }
- } else {
- // if blocks is already loaded then directly set the block at particular position
- //so block will be present in sorted order
- loadedBlock[counter] = tableBlock;
- }
- }
- // shutdown the executor gracefully and wait until all the task is finished
- executor.shutdown();
- try {
- executor.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new IndexBuilderException(e);
- }
- // fill the block which were not loaded before to loaded blocks array
- fillLoadedBlocks(loadedBlock, blocksList);
- return Arrays.asList(loadedBlock);
- }
-
- /**
- * Below method will be used to fill the loaded blocks to the array
- * which will be used for query execution
- *
- * @param loadedBlockArray array of blocks which will be filled
- * @param blocksList blocks loaded in thread
- * @throws IndexBuilderException in case of any failure
- */
- private void fillLoadedBlocks(AbstractIndex[] loadedBlockArray,
- List<Future<AbstractIndex>> blocksList) throws IndexBuilderException {
- int blockCounter = 0;
- for (int i = 0; i < loadedBlockArray.length; i++) {
- if (null == loadedBlockArray[i]) {
- try {
- loadedBlockArray[i] = blocksList.get(blockCounter++).get();
- } catch (InterruptedException | ExecutionException e) {
- throw new IndexBuilderException(e);
- }
- }
-
- }
- }
-
- private AbstractIndex loadBlock(Map<TableBlockInfo, AbstractIndex> tableBlockMapTemp,
- TableBlockInfo blockInfo) throws CarbonUtilException {
- AbstractIndex tableBlock;
- DataFileFooter footer;
- // getting the data file meta data of the block
- footer = CarbonUtil.readMetadatFile(blockInfo.getFilePath(), blockInfo.getBlockOffset(),
- blockInfo.getBlockLength());
- tableBlock = new BlockIndex();
- footer.setTableBlockInfo(blockInfo);
- // building the block
- tableBlock.buildIndex(Arrays.asList(footer));
- tableBlockMapTemp.put(blockInfo, tableBlock);
- // finally remove the lock object from block info lock as once block is loaded
- // it will not come inside this if condition
- blockInfoLock.remove(blockInfo);
- return tableBlock;
- }
-
- /**
- * Method to add table level lock if lock is not present for the table
- *
- * @param absoluteTableIdentifier
- */
- private synchronized void addTableLockObject(AbsoluteTableIdentifier absoluteTableIdentifier) {
- // add the instance to lock map if it is not present
- if (null == tableLockMap.get(absoluteTableIdentifier)) {
- tableLockMap.put(absoluteTableIdentifier, new Object());
- }
- }
-
- /**
- * This will be used to remove a particular blocks useful in case of
- * deletion of some of the blocks in case of retention or may be some other
- * scenario
- *
- * @param removeTableBlocksInfos blocks to be removed
- * @param absoluteTableIdentifier absolute table identifier
- */
- public void removeTableBlocks(List<TableBlockInfo> removeTableBlocksInfos,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- // get the lock object if lock object is not present then it is not
- // loaded at all
- // we can return from here
- Object lockObject = tableLockMap.get(absoluteTableIdentifier);
- if (null == lockObject) {
- return;
- }
- Map<TableBlockInfo, AbstractIndex> map = tableBlocksMap.get(absoluteTableIdentifier);
- // if there is no loaded blocks then return
- if (null == map) {
- return;
- }
- for (TableBlockInfo blockInfos : removeTableBlocksInfos) {
- map.remove(blockInfos);
- }
- }
-
- /**
- * remove all the details of a table this will be used in case of drop table
- *
- * @param absoluteTableIdentifier absolute table identifier to find the table
- */
- public void clear(AbsoluteTableIdentifier absoluteTableIdentifier) {
- // removing all the details of table
- tableLockMap.remove(absoluteTableIdentifier);
- tableBlocksMap.remove(absoluteTableIdentifier);
- }
-
- /**
- * Thread class which will be used to load the blocks
- */
- private class BlockLoaderThread implements Callable<AbstractIndex> {
- /**
- * table block info to block index map
- */
- private Map<TableBlockInfo, AbstractIndex> tableBlockMap;
-
- // block info
- private TableBlockInfo blockInfo;
-
- private BlockLoaderThread(TableBlockInfo blockInfo,
- Map<TableBlockInfo, AbstractIndex> tableBlockMap) {
- this.tableBlockMap = tableBlockMap;
- this.blockInfo = blockInfo;
- }
-
- @Override public AbstractIndex call() throws Exception {
- // load and return the loaded blocks
- return loadBlock(tableBlockMap, blockInfo);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/BtreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/BtreeBuilder.java b/core/src/main/java/org/carbondata/core/carbon/datastore/BtreeBuilder.java
deleted file mode 100644
index f6ea7d0..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/BtreeBuilder.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-/**
- * Below interface will be used to build the index
- * in some data structure
- */
-public interface BtreeBuilder {
-
- /**
- * Below method will be used to store the leaf collection in some data structure
- */
- void build(BTreeBuilderInfo blocksBuilderInfos);
-
- /**
- * below method to get the first data block
- *
- * @return data block
- */
- DataRefNode get();
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNode.java b/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNode.java
deleted file mode 100644
index 4bb551f..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNode.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-import org.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import org.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
-import org.carbondata.core.datastorage.store.FileHolder;
-
-/**
- * Interface data block reference
- */
-public interface DataRefNode {
-
- /**
- * Method to get the next block this can be used while scanning when
- * iterator of this class can be used iterate over blocks
- *
- * @return next block
- */
- DataRefNode getNextDataRefNode();
-
- /**
- * to get the number of keys tuples present in the block
- *
- * @return number of keys in the block
- */
- int nodeSize();
-
- /**
- * Method can be used to get the block index .This can be used when multiple
- * thread can be used scan group of blocks in that can we can assign the
- * some of the blocks to one thread and some to other
- *
- * @return block number
- */
- long nodeNumber();
-
- /**
- * This method will be used to get the max value of all the columns this can
- * be used in case of filter query
- *
- * @param max value of all the columns
- */
- byte[][] getColumnsMaxValue();
-
- /**
- * This method will be used to get the min value of all the columns this can
- * be used in case of filter query
- *
- * @param min value of all the columns
- */
- byte[][] getColumnsMinValue();
-
- /**
- * Below method will be used to get the dimension chunks
- *
- * @param fileReader file reader to read the chunks from file
- * @param blockIndexes indexes of the blocks need to be read
- * @return dimension data chunks
- */
- DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, int[] blockIndexes);
-
- /**
- * Below method will be used to get the dimension chunk
- *
- * @param fileReader file reader to read the chunk from file
- * @param blockIndex block index to be read
- * @return dimension data chunk
- */
- DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, int blockIndexes);
-
- /**
- * Below method will be used to get the measure chunk
- *
- * @param fileReader file reader to read the chunk from file
- * @param blockIndexes block indexes to be read from file
- * @return measure column data chunk
- */
- MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, int[] blockIndexes);
-
- /**
- * Below method will be used to read the measure chunk
- *
- * @param fileReader file read to read the file chunk
- * @param blockIndex block index to be read from file
- * @return measure data chunk
- */
- MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNodeFinder.java b/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNodeFinder.java
deleted file mode 100644
index a4027cf..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/DataRefNodeFinder.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-/**
- * Below Interface is to search a block
- */
-public interface DataRefNodeFinder {
-
- /**
- * Below method will be used to get the first tentative block which matches with
- * the search key
- *
- * @param dataBlocks complete data blocks present
- * @param serachKey key to be search
- * @return data block
- */
- DataRefNode findFirstDataBlock(DataRefNode dataBlocks, IndexKey searchKey);
-
- /**
- * Below method will be used to get the last tentative block which matches with
- * the search key
- *
- * @param dataBlocks complete data blocks present
- * @param serachKey key to be search
- * @return data block
- */
- DataRefNode findLastDataBlock(DataRefNode dataBlocks, IndexKey searchKey);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/IndexKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/IndexKey.java b/core/src/main/java/org/carbondata/core/carbon/datastore/IndexKey.java
deleted file mode 100644
index 49d443c..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/IndexKey.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-/**
- * Index class to store the index of the segment blocklet infos
- */
-public class IndexKey {
-
- /**
- * key which is generated from key generator
- */
- private byte[] dictionaryKeys;
-
- /**
- * key which was no generated using key generator
- * <Index of FirstKey (2 bytes)><Index of SecondKey (2 bytes)><Index of NKey (2 bytes)>
- * <First Key ByteArray><2nd Key ByteArray><N Key ByteArray>
- */
- private byte[] noDictionaryKeys;
-
- public IndexKey(byte[] dictionaryKeys, byte[] noDictionaryKeys) {
- this.dictionaryKeys = dictionaryKeys;
- this.noDictionaryKeys = noDictionaryKeys;
- if (null == dictionaryKeys) {
- this.dictionaryKeys = new byte[0];
- }
- if (null == noDictionaryKeys) {
- this.noDictionaryKeys = new byte[0];
- }
- }
-
- /**
- * @return the dictionaryKeys
- */
- public byte[] getDictionaryKeys() {
- return dictionaryKeys;
- }
-
- /**
- * @return the noDictionaryKeys
- */
- public byte[] getNoDictionaryKeys() {
- return noDictionaryKeys;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
deleted file mode 100644
index c94a100..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.carbondata.core.carbon.datastore.block.AbstractIndex;
-import org.carbondata.core.carbon.datastore.block.SegmentTaskIndex;
-import org.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.carbondata.core.carbon.datastore.exception.IndexBuilderException;
-import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.carbondata.core.carbon.path.CarbonTablePath.DataFileUtil;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.util.CarbonUtilException;
-
-/**
- * Singleton Class to handle loading, unloading,clearing,storing of the table
- * blocks
- */
-public class SegmentTaskIndexStore {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(SegmentTaskIndexStore.class.getName());
- /**
- * singleton instance
- */
- private static final SegmentTaskIndexStore SEGMENTTASKINDEXSTORE = new SegmentTaskIndexStore();
-
- /**
- * mapping of table identifier to map of segmentId_taskId to table segment
- * reason of so many map as each segment can have multiple data file and
- * each file will have its own btree
- */
- private Map<AbsoluteTableIdentifier, Map<String, Map<String, AbstractIndex>>> tableSegmentMap;
-
- /**
- * map of block info to lock object map, while loading the btree this will be filled
- * and removed after loading the tree for that particular block info, this will be useful
- * while loading the tree concurrently so only block level lock will be applied another
- * block can be loaded concurrently
- */
- private Map<String, Object> segmentLockMap;
-
- /**
- * table and its lock object to this will be useful in case of concurrent
- * query scenario when more than one query comes for same table and in that
- * case it will ensure that only one query will able to load the blocks
- */
- private Map<AbsoluteTableIdentifier, Object> tableLockMap;
-
- private SegmentTaskIndexStore() {
- tableSegmentMap =
- new ConcurrentHashMap<AbsoluteTableIdentifier, Map<String, Map<String, AbstractIndex>>>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- tableLockMap = new ConcurrentHashMap<AbsoluteTableIdentifier, Object>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- segmentLockMap = new ConcurrentHashMap<String, Object>();
- }
-
- /**
- * Return the instance of this class
- *
- * @return singleton instance
- */
- public static SegmentTaskIndexStore getInstance() {
- return SEGMENTTASKINDEXSTORE;
- }
-
- /**
- * Below method will be used to load the segment of segments
- * One segment may have multiple task , so table segment will be loaded
- * based on task id and will return the map of taksId to table segment
- * map
- *
- * @param segmentToTableBlocksInfos segment id to block info
- * @param absoluteTableIdentifier absolute table identifier
- * @return map of taks id to segment mapping
- * @throws IndexBuilderException
- */
- public Map<String, AbstractIndex> loadAndGetTaskIdToSegmentsMap(
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IndexBuilderException {
- // task id to segment map
- Map<String, AbstractIndex> taskIdToTableSegmentMap =
- new HashMap<String, AbstractIndex>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- addLockObject(absoluteTableIdentifier);
- Iterator<Entry<String, List<TableBlockInfo>>> iteratorOverSegmentBlocksInfos =
- segmentToTableBlocksInfos.entrySet().iterator();
- Map<String, Map<String, AbstractIndex>> tableSegmentMapTemp =
- addTableSegmentMap(absoluteTableIdentifier);
- Map<String, AbstractIndex> taskIdToSegmentIndexMap = null;
- String segmentId = null;
- String taskId = null;
- try {
- while (iteratorOverSegmentBlocksInfos.hasNext()) {
- // segment id to table block mapping
- Entry<String, List<TableBlockInfo>> next = iteratorOverSegmentBlocksInfos.next();
- // group task id to table block info mapping for the segment
- Map<String, List<TableBlockInfo>> taskIdToTableBlockInfoMap =
- mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos);
- // get the existing map of task id to table segment map
- segmentId = next.getKey();
- // check if segment is already loaded, if segment is already loaded
- //no need to load the segment block
- taskIdToSegmentIndexMap = tableSegmentMapTemp.get(segmentId);
- if (taskIdToSegmentIndexMap == null) {
- // get the segment loader lock object this is to avoid
- // same segment is getting loaded multiple times
- // in case of concurrent query
- Object segmentLoderLockObject = segmentLockMap.get(segmentId);
- if (null == segmentLoderLockObject) {
- segmentLoderLockObject = addAndGetSegmentLock(segmentId);
- }
- // acquire lock to lod the segment
- synchronized (segmentLoderLockObject) {
- taskIdToSegmentIndexMap = tableSegmentMapTemp.get(segmentId);
- if (null == taskIdToSegmentIndexMap) {
- // creating a map of take if to table segment
- taskIdToSegmentIndexMap = new HashMap<String, AbstractIndex>();
- Iterator<Entry<String, List<TableBlockInfo>>> iterator =
- taskIdToTableBlockInfoMap.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<String, List<TableBlockInfo>> taskToBlockInfoList = iterator.next();
- taskId = taskToBlockInfoList.getKey();
- taskIdToSegmentIndexMap.put(taskId,
- loadBlocks(taskId, taskToBlockInfoList.getValue(), absoluteTableIdentifier));
- }
- tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap);
- // removing from segment lock map as once segment is loaded
- //if concurrent query is coming for same segment
- // it will wait on the lock so after this segment will be already
- //loaded so lock is not required, that is why removing the
- // the lock object as it wont be useful
- segmentLockMap.remove(segmentId);
- }
- }
- taskIdToTableSegmentMap.putAll(taskIdToSegmentIndexMap);
- }
- }
- } catch (CarbonUtilException e) {
- LOGGER.error("Problem while loading the segment");
- throw new IndexBuilderException(e);
- }
- return taskIdToTableSegmentMap;
- }
-
- /**
- * Below method will be used to get the segment level lock object
- *
- * @param segmentId
- * @return lock object
- */
- private synchronized Object addAndGetSegmentLock(String segmentId) {
- // get the segment lock object if it is present then return
- // otherwise add the new lock and return
- Object segmentLoderLockObject = segmentLockMap.get(segmentId);
- if (null == segmentLoderLockObject) {
- segmentLoderLockObject = new Object();
- segmentLockMap.put(segmentId, segmentLoderLockObject);
- }
- return segmentLoderLockObject;
- }
-
- /**
- * Below code is to add table lock map which will be used to
- * add
- *
- * @param absoluteTableIdentifier
- */
- private synchronized void addLockObject(AbsoluteTableIdentifier absoluteTableIdentifier) {
- // add the instance to lock map if it is not present
- if (null == tableLockMap.get(absoluteTableIdentifier)) {
- tableLockMap.put(absoluteTableIdentifier, new Object());
- }
- }
-
- /**
- * Below method will be used to get the table segment map
- * if table segment is not present then it will add and return
- *
- * @param absoluteTableIdentifier
- * @return table segment map
- */
- private Map<String, Map<String, AbstractIndex>> addTableSegmentMap(
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- // get the instance of lock object
- Object lockObject = tableLockMap.get(absoluteTableIdentifier);
- Map<String, Map<String, AbstractIndex>> tableSegmentMapTemp =
- tableSegmentMap.get(absoluteTableIdentifier);
- if (null == tableSegmentMapTemp) {
- synchronized (lockObject) {
- // segment id to task id to table segment map
- tableSegmentMapTemp = tableSegmentMap.get(absoluteTableIdentifier);
- if (null == tableSegmentMapTemp) {
- tableSegmentMapTemp = new ConcurrentHashMap<String, Map<String, AbstractIndex>>();
- tableSegmentMap.put(absoluteTableIdentifier, tableSegmentMapTemp);
- }
- }
- }
- return tableSegmentMapTemp;
- }
-
- /**
- * Below method will be used to load the blocks
- *
- * @param tableBlockInfoList
- * @return loaded segment
- * @throws CarbonUtilException
- */
- private AbstractIndex loadBlocks(String taskId, List<TableBlockInfo> tableBlockInfoList,
- AbsoluteTableIdentifier tableIdentifier) throws CarbonUtilException {
- // all the block of one task id will be loaded together
- // so creating a list which will have all the data file meta data to of one task
- List<DataFileFooter> footerList =
- CarbonUtil.readCarbonIndexFile(taskId, tableBlockInfoList, tableIdentifier);
- AbstractIndex segment = new SegmentTaskIndex();
- // file path of only first block is passed as it all table block info path of
- // same task id will be same
- segment.buildIndex(footerList);
- return segment;
- }
-
- /**
- * Below method will be used to get the task id to all the table block info belongs to
- * that task id mapping
- *
- * @param segmentToTableBlocksInfos segment if to table blocks info map
- * @return task id to table block info mapping
- */
- private Map<String, List<TableBlockInfo>> mappedAndGetTaskIdToTableBlockInfo(
- Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos) {
- Map<String, List<TableBlockInfo>> taskIdToTableBlockInfoMap =
- new HashMap<String, List<TableBlockInfo>>();
- Iterator<Entry<String, List<TableBlockInfo>>> iterator =
- segmentToTableBlocksInfos.entrySet().iterator();
- while (iterator.hasNext()) {
- Entry<String, List<TableBlockInfo>> next = iterator.next();
- List<TableBlockInfo> value = next.getValue();
- for (TableBlockInfo blockInfo : value) {
- String taskNo = DataFileUtil.getTaskNo(blockInfo.getFilePath());
- List<TableBlockInfo> list = taskIdToTableBlockInfoMap.get(taskNo);
- if (null == list) {
- list = new ArrayList<TableBlockInfo>();
- taskIdToTableBlockInfoMap.put(taskNo, list);
- }
- list.add(blockInfo);
- }
-
- }
- return taskIdToTableBlockInfoMap;
- }
-
- /**
- * remove all the details of a table this will be used in case of drop table
- *
- * @param absoluteTableIdentifier absolute table identifier to find the table
- */
- public void clear(AbsoluteTableIdentifier absoluteTableIdentifier) {
- // removing all the details of table
- tableLockMap.remove(absoluteTableIdentifier);
- tableSegmentMap.remove(absoluteTableIdentifier);
- }
-
- /**
- * Below method will be used to remove the segment block based on
- * segment id is passed
- *
- * @param segmentToBeRemoved segment to be removed
- * @param absoluteTableIdentifier absoluteTableIdentifier
- */
- public void removeTableBlocks(List<String> segmentToBeRemoved,
- AbsoluteTableIdentifier absoluteTableIdentifier) {
- // get the lock object if lock object is not present then it is not
- // loaded at all
- // we can return from here
- Object lockObject = tableLockMap.get(absoluteTableIdentifier);
- if (null == lockObject) {
- return;
- }
- // Acquire the lock and remove only those instance which was loaded
- Map<String, Map<String, AbstractIndex>> map = tableSegmentMap.get(absoluteTableIdentifier);
- // if there is no loaded blocks then return
- if (null == map) {
- return;
- }
- for (String segmentId : segmentToBeRemoved) {
- map.remove(segmentId);
- }
- }
-
- /**
- * Below method will be used to check if segment blocks
- * is already loaded or not
- *
- * @param absoluteTableIdentifier
- * @param segmentId
- * @return is loaded then return the loaded blocks otherwise null
- */
- public Map<String, AbstractIndex> getSegmentBTreeIfExists(
- AbsoluteTableIdentifier absoluteTableIdentifier, String segmentId) {
- Map<String, Map<String, AbstractIndex>> tableSegment =
- tableSegmentMap.get(absoluteTableIdentifier);
- if (null == tableSegment) {
- return null;
- }
- return tableSegment.get(segmentId);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/block/AbstractIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/block/AbstractIndex.java b/core/src/main/java/org/carbondata/core/carbon/datastore/block/AbstractIndex.java
deleted file mode 100644
index 548522d..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/AbstractIndex.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore.block;
-
-import java.util.List;
-
-import org.carbondata.core.carbon.datastore.DataRefNode;
-import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-
-public abstract class AbstractIndex {
-
- /**
- * vo class which will hold the RS information of the block
- */
- protected SegmentProperties segmentProperties;
-
- /**
- * data block
- */
- protected DataRefNode dataRefNode;
-
- /**
- * total number of row present in the block
- */
- protected long totalNumberOfRows;
-
- /**
- * @return the totalNumberOfRows
- */
- public long getTotalNumberOfRows() {
- return totalNumberOfRows;
- }
-
- /**
- * @return the segmentProperties
- */
- public SegmentProperties getSegmentProperties() {
- return segmentProperties;
- }
-
- /**
- * @return the dataBlock
- */
- public DataRefNode getDataRefNode() {
- return dataRefNode;
- }
-
- /**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- public abstract void buildIndex(List<DataFileFooter> footerList);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/block/BlockIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/block/BlockIndex.java b/core/src/main/java/org/carbondata/core/carbon/datastore/block/BlockIndex.java
deleted file mode 100644
index 3ace21d..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/BlockIndex.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.carbon.datastore.block;
-
-import java.util.List;
-
-import org.carbondata.core.carbon.datastore.BTreeBuilderInfo;
-import org.carbondata.core.carbon.datastore.BtreeBuilder;
-import org.carbondata.core.carbon.datastore.impl.btree.BlockletBTreeBuilder;
-import org.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-
-/**
- * Class which is responsible for loading the b+ tree block. This class will
- * persist all the detail of a table block
- */
-public class BlockIndex extends AbstractIndex {
-
- /**
- * Below method will be used to load the data block
- *
- * @param blockInfo block detail
- */
- public void buildIndex(List<DataFileFooter> footerList) {
- // create a metadata details
- // this will be useful in query handling
- segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(),
- footerList.get(0).getSegmentInfo().getColumnCardinality());
- // create a segment builder info
- BTreeBuilderInfo indexBuilderInfo =
- new BTreeBuilderInfo(footerList, segmentProperties.getDimensionColumnsValueSize());
- BtreeBuilder blocksBuilder = new BlockletBTreeBuilder();
- // load the metadata
- blocksBuilder.build(indexBuilderInfo);
- dataRefNode = blocksBuilder.get();
- totalNumberOfRows = footerList.get(0).getNumberOfRows();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/carbon/datastore/block/Distributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/datastore/block/Distributable.java b/core/src/main/java/org/carbondata/core/carbon/datastore/block/Distributable.java
deleted file mode 100644
index 977ee3a..0000000
--- a/core/src/main/java/org/carbondata/core/carbon/datastore/block/Distributable.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.carbondata.core.carbon.datastore.block;
-
-/**
- * Abstract class which is maintains the locations of node.
- */
-public abstract class Distributable implements Comparable<Distributable> {
-
- public abstract String[] getLocations();
-}