You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/05/15 05:06:23 UTC
[10/19] carbondata git commit: [CARBONDATA-935] Define PartitionInfo
model
[CARBONDATA-935] Define PartitionInfo model
fix javastyle error
modify partitioninfo and singlePartition
modify schema.thrift to add partitioningList
modify definition
variable style fix, remove partitionName and some comments
change import order
change comments
change partition info and create partition statistic
add get method in carbontable
remove partition in schema
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f4d081e8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f4d081e8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f4d081e8
Branch: refs/heads/master
Commit: f4d081e81714d2d6795af1eab64a68164a4711dd
Parents: bf44c9f
Author: lucao <wh...@gmail.com>
Authored: Tue Apr 18 14:52:18 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sun May 14 20:40:13 2017 +0800
----------------------------------------------------------------------
.../core/metadata/schema/PartitionInfo.java | 98 ++++++++++++++++++++
.../schema/partition/AbstractPartition.java | 42 +++++++++
.../schema/partition/HashPartition.java | 34 +++++++
.../schema/partition/ListPartition.java | 36 +++++++
.../schema/partition/PartitionType.java | 27 ++++++
.../schema/partition/RangePartition.java | 40 ++++++++
.../core/metadata/schema/table/CarbonTable.java | 20 ++++
.../core/metadata/schema/table/TableSchema.java | 14 +++
.../core/stats/PartitionStatistic.java | 59 ++++++++++++
format/src/main/thrift/schema.thrift | 18 ++++
10 files changed, 388 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
new file mode 100644
index 0000000..86ef3c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+/**
+ * Partition Information of carbon partition table
+ */
+public class PartitionInfo implements Serializable {
+
+ /**
+ * Partition columns
+ */
+ private List<ColumnSchema> columnSchemaList;
+
+ /**
+ * partition type
+ */
+ private PartitionType partitionType;
+
+ /**
+ * Range Partition definition
+ */
+ private List<String> rangeInfo;
+
+ /**
+ * List Partition definition
+ */
+ private List<List<String>> listInfo;
+
+ /**
+ * Hash Partition numbers
+ */
+ private int hashNumber;
+
+ /**
+ * For range partition table
+ * @param columnSchemaList
+ * @param partitionType
+ */
+ public PartitionInfo(List<ColumnSchema> columnSchemaList, PartitionType partitionType) {
+ this.columnSchemaList = columnSchemaList;
+ this.partitionType = partitionType;
+ }
+
+ public List<ColumnSchema> getColumnSchemaList() {
+ return columnSchemaList;
+ }
+
+ public PartitionType getPartitionType() {
+ return partitionType;
+ }
+
+ public void setHashNumber(int hashNumber) {
+ this.hashNumber = hashNumber;
+ }
+
+ public int getHashNumber() {
+ return hashNumber;
+ }
+
+ public void setRangeInfo(List<String> rangeInfo) {
+ this.rangeInfo = rangeInfo;
+ }
+
+ public List<String> getRangeInfo() {
+ return rangeInfo;
+ }
+
+ public void setListInfo(List<List<String>> listInfo) {
+ this.listInfo = listInfo;
+ }
+
+ public List<List<String>> getListInfo() {
+ return listInfo;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
new file mode 100644
index 0000000..1d6337c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public abstract class AbstractPartition {
+
+ /**
+ * Partition unique identifier
+ */
+ protected int partitionId;
+
+ /**
+ * Total row count of this partition
+ */
+ protected int rowCount;
+
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ public void setRowCount(int count) {
+ this.rowCount = count;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
new file mode 100644
index 0000000..6b43525
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class HashPartition extends AbstractPartition {
+
+ /**
+ * hash value for hash partition table
+ */
+ private int hashValue;
+
+ public HashPartition(int id, int hashValue) {
+ this.partitionId = id;
+ this.hashValue = hashValue;
+ }
+
+ public int getHashValue() {
+ return hashValue;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
new file mode 100644
index 0000000..11a396f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+import java.util.List;
+
+public class ListPartition extends AbstractPartition {
+
+ /**
+ * value list for list partition table
+ */
+ private List<String> listInfo;
+
+ public ListPartition(int id, List<String> listInfo) {
+ this.partitionId = id;
+ this.listInfo = listInfo;
+ }
+
+ public List<String> getListInfo() {
+ return listInfo;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
new file mode 100644
index 0000000..9a4cf4a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+/**
+ * Partition type supported in carbon
+ */
+public enum PartitionType {
+ RANGE,
+ RANGE_INTERVAL,
+ LIST,
+ HASH
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
new file mode 100644
index 0000000..af2b05d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class RangePartition extends AbstractPartition {
+ /**
+ * boundary value for range partition table
+ */
+ private String lowerBoundary;
+
+ private String upperBoundary;
+
+ public RangePartition(int id, String lowerBoundary, String upperBoundary) {
+ this.partitionId = id;
+ this.lowerBoundary = lowerBoundary;
+ this.upperBoundary = upperBoundary;
+ }
+
+ public String getLowerBoundary() {
+ return lowerBoundary;
+ }
+
+ public String getUpperBoundary() {
+ return upperBoundary;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index fae9960..ce5cbe4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -27,11 +27,13 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.stats.PartitionStatistic;
/**
* Mapping class for Carbon actual table
@@ -92,6 +94,15 @@ public class CarbonTable implements Serializable {
private Map<String, BucketingInfo> tableBucketMap;
/**
+ * table partition info
+ */
+ private Map<String, PartitionInfo> tablePartitionMap;
+
+ /**
+ * statistic information of partition table
+ */
+ private PartitionStatistic partitionStatistic;
+ /**
* tableUniqueName
*/
private String tableUniqueName;
@@ -131,6 +142,7 @@ public class CarbonTable implements Serializable {
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
this.tableBucketMap = new HashMap<>();
+ this.tablePartitionMap = new HashMap<>();
this.aggregateTablesName = new ArrayList<String>();
this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
this.tablePrimitiveDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -566,6 +578,14 @@ public class CarbonTable implements Serializable {
return tableBucketMap.get(tableName);
}
+ public PartitionInfo getPartitionInfo(String tableName) {
+ return tablePartitionMap.get(tableName);
+ }
+
+ public PartitionStatistic getPartitionStatistic() {
+ return partitionStatistic;
+ }
+
/**
* @return absolute table identifier
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 179f4d4..ad2730b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -66,6 +67,11 @@ public class TableSchema implements Serializable {
*/
private BucketingInfo bucketingInfo;
+ /**
+ * Information about partition type, partition column, numbers
+ */
+ private PartitionInfo partitionInfo;
+
public TableSchema() {
this.listOfColumns = new ArrayList<ColumnSchema>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
@@ -184,4 +190,12 @@ public class TableSchema implements Serializable {
public void setBucketingInfo(BucketingInfo bucketingInfo) {
this.bucketingInfo = bucketingInfo;
}
+
+ public PartitionInfo getPartitionInfo() {
+ return partitionInfo;
+ }
+
+ public void setpartitionInfo(PartitionInfo partitionInfo) {
+ this.partitionInfo = partitionInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
new file mode 100644
index 0000000..44f62b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.schema.partition.AbstractPartition;
+
+public class PartitionStatistic implements Serializable {
+
+ /**
+ * total count of partitions
+ */
+ private int numberOfPartitions;
+
+ /**
+ * partition id, increase only
+ */
+ private int partitionIndex;
+
+ private Map<Integer, AbstractPartition> partitionMap;
+
+ public PartitionStatistic() {
+ this.partitionIndex = 0;
+ this.numberOfPartitions = 0;
+ this.partitionMap = new HashMap<>();
+ }
+
+ public void addNewPartition(int id, AbstractPartition partition) {
+ partitionMap.put(id, partition);
+ partitionIndex ++;
+ numberOfPartitions ++;
+ }
+
+ public void deletePartition(int id) {
+ partitionMap.remove(id);
+ numberOfPartitions --;
+ }
+
+ public int getNumberOfPartitions() {
+ return numberOfPartitions;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4d081e8/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index fba70ea..695333c 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -49,6 +49,12 @@ enum Encoding{
DIRECT_DICTIONARY = 5; // Identifies that a column is direct dictionary encoded
}
+enum PartitionType{
+ RANGE = 0;
+ RANGE_INTERVAL = 1;
+ LIST = 2;
+ HASH = 3;
+}
/**
* Description of a Column for both dimension and measure
@@ -121,6 +127,17 @@ struct SchemaEvolution{
}
/**
+ * Partition information of table
+ */
+struct PartitionInfo{
+ 1: required list<ColumnSchema> partition_columns;
+ 2: required PartitionType partition_type;
+ 3: optional i32 hash_number; // number of partitions defined in hash partition table
+ 4: optional list<list<string>> list_info; // value list of list partition table
+ 5: optional list<string> range_info; // range value list of range partition table
+}
+
+/**
* Bucketing information of fields on table
*/
struct BucketingInfo{
@@ -137,6 +154,7 @@ struct TableSchema{
3: required SchemaEvolution schema_evolution; // History of schema evolution of this table
4: optional map<string,string> tableProperties; // Table properties configured by the user
5: optional BucketingInfo bucketingInfo; // Bucketing information
+ 6: optional PartitionInfo partitionInfo; // Partition information
}
struct TableInfo{