You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by bl...@apache.org on 2013/12/11 07:16:48 UTC
git commit: TAJO-405: Improve HCatalogStore to support partitioned
table. (jaehwa)
Updated Branches:
refs/heads/master 01c25d28c -> 93bae895f
TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/93bae895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/93bae895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/93bae895
Branch: refs/heads/master
Commit: 93bae895f539ca1c787150210ce78b2d3233d366
Parents: 01c25d2
Author: blrunner <jh...@gruter.com>
Authored: Wed Dec 11 15:16:31 2013 +0900
Committer: blrunner <jh...@gruter.com>
Committed: Wed Dec 11 15:16:31 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../tajo-catalog-drivers/tajo-hcatalog/pom.xml | 33 +++++++++++++++++---
.../tajo/catalog/store/HCatalogStore.java | 23 ++++++++++++--
3 files changed, 50 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b32a5a5..1e662ef 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -86,6 +86,8 @@ Release 0.8.0 - unreleased
IMPROVEMENTS
+ TAJO-405: Improve HCatalogStore to support partitioned table. (jaehwa)
+
TAJO-336: Separate catalog stores into separate modules. (jaehwa)
TAJO-329: Implement physical operator to store in column-partitioned table. (jaehwa)
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
index 738a9f7..aeb4a08 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/pom.xml
@@ -94,10 +94,7 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-rpc</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.tajo</groupId>
- <artifactId>tajo-algebra</artifactId>
- </dependency>
+
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
@@ -121,7 +118,6 @@
</dependency>
</dependencies>
-
<profiles>
<profile>
<id>hcatalog-0.11.0</id>
@@ -368,6 +364,33 @@
</dependencies>
</profile>
<profile>
+ <id>docs</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <executions>
+ <execution>
+ <!-- build javadoc jars per jar for publishing to maven -->
+ <id>module-javadocs</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ <configuration>
+ <destDir>${project.build.directory}</destDir>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>src</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/93bae895/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index d8a0e98..c41a690 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -30,7 +30,7 @@ import org.apache.hcatalog.data.Pair;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.tajo.catalog.*;
-import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.partition.Partitions;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes;
@@ -103,9 +103,10 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
HiveMetaStoreClient client = null;
Path path = null;
CatalogProtos.StoreType storeType = null;
- Schema schema = null;
+ org.apache.tajo.catalog.Schema schema = null;
Options options = null;
TableStats stats = null;
+ Partitions partitions = null;
// get db name and table name.
try {
@@ -132,7 +133,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
// convert hcatalog field schema into tajo field schema.
- schema = new Schema();
+ schema = new org.apache.tajo.catalog.Schema();
HCatSchema tableSchema = HCatUtil.getTableSchemaWithPtnCols(table);
List<HCatFieldSchema> fieldSchemaList = tableSchema.getFields();
for (HCatFieldSchema eachField : fieldSchemaList) {
@@ -174,6 +175,18 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
}
}
+ if (table.getPartitionKeys() != null) {
+ partitions = new Partitions();
+ List<FieldSchema> partitionKeys = table.getPartitionKeys();
+ for(int i = 0; i < partitionKeys.size(); i++) {
+ FieldSchema fieldSchema = partitionKeys.get(i);
+ TajoDataTypes.Type dataType = HCatalogUtil.getTajoFieldType(fieldSchema.getType().toString());
+ partitions.addColumn(new Column(fieldSchema.getName(), dataType));
+ }
+ partitions.setPartitionsType(CatalogProtos.PartitionsType.COLUMN);
+ }
+
+
} finally {
HCatUtil.closeHiveClientQuietly(client);
}
@@ -183,10 +196,14 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
if (stats != null) {
tableDesc.setStats(stats);
}
+ if (partitions != null) {
+ tableDesc.setPartitions(partitions);
+ }
return tableDesc;
}
+
private TajoDataTypes.Type getDataType(final String typeStr) {
try {
return Enum.valueOf(TajoDataTypes.Type.class, typeStr);