You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/07/19 19:55:30 UTC
[31/51] [partial] hive git commit: HIVE-20188 : Split server-specific
code outside of standalone metastore-common (Alexander Kolbasov reviewed by
Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java
deleted file mode 100644
index f61a62c..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SchemaCompatibility;
-import org.apache.hadoop.hive.metastore.api.SchemaType;
-import org.apache.hadoop.hive.metastore.api.SchemaValidation;
-
-public class ISchemaBuilder {
- private SchemaType schemaType; // required
- private String name; // required
- private String dbName, catName; // required
- private SchemaCompatibility compatibility; // required
- private SchemaValidation validationLevel; // required
- private boolean canEvolve; // required
- private String schemaGroup; // optional
- private String description; // optional
-
- public ISchemaBuilder() {
- compatibility = SchemaCompatibility.BACKWARD;
- validationLevel = SchemaValidation.ALL;
- canEvolve = true;
- dbName = Warehouse.DEFAULT_DATABASE_NAME;
- catName = Warehouse.DEFAULT_CATALOG_NAME;
- }
-
- public ISchemaBuilder setSchemaType(SchemaType schemaType) {
- this.schemaType = schemaType;
- return this;
- }
-
- public ISchemaBuilder setName(String name) {
- this.name = name;
- return this;
- }
-
- public ISchemaBuilder setDbName(String dbName) {
- this.dbName = dbName;
- return this;
- }
-
- public ISchemaBuilder inDb(Database db) {
- this.catName = db.getCatalogName();
- this.dbName = db.getName();
- return this;
- }
-
- public ISchemaBuilder setCompatibility(SchemaCompatibility compatibility) {
- this.compatibility = compatibility;
- return this;
- }
-
- public ISchemaBuilder setValidationLevel(SchemaValidation validationLevel) {
- this.validationLevel = validationLevel;
- return this;
- }
-
- public ISchemaBuilder setCanEvolve(boolean canEvolve) {
- this.canEvolve = canEvolve;
- return this;
- }
-
- public ISchemaBuilder setSchemaGroup(String schemaGroup) {
- this.schemaGroup = schemaGroup;
- return this;
- }
-
- public ISchemaBuilder setDescription(String description) {
- this.description = description;
- return this;
- }
-
- public ISchema build() throws MetaException {
- if (schemaType == null || name == null) {
- throw new MetaException("You must provide a schemaType and name");
- }
- ISchema iSchema =
- new ISchema(schemaType, name, catName, dbName, compatibility, validationLevel, canEvolve);
- if (schemaGroup != null) iSchema.setSchemaGroup(schemaGroup);
- if (description != null) iSchema.setDescription(description);
- return iSchema;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java
deleted file mode 100644
index d6ee673..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.thrift.TException;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Builder for {@link Partition}. The only requirements are 1. (database name and table name) or table
- * reference; 2. partition values; 3. whatever {@link StorageDescriptorBuilder} requires.
- */
-public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> {
- private String catName, dbName, tableName;
- private int createTime, lastAccessTime;
- private Map<String, String> partParams;
- private List<String> values;
-
- public PartitionBuilder() {
- // Set some reasonable defaults
- partParams = new HashMap<>();
- createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000);
- dbName = Warehouse.DEFAULT_DATABASE_NAME;
- super.setChild(this);
- }
-
- public PartitionBuilder setDbName(String dbName) {
- this.dbName = dbName;
- return this;
- }
-
- public PartitionBuilder setTableName(String tableName) {
- this.tableName = tableName;
- return this;
- }
-
- public PartitionBuilder inTable(Table table) {
- this.dbName = table.getDbName();
- this.tableName = table.getTableName();
- this.catName = table.getCatName();
- setCols(table.getSd().getCols());
- return this;
- }
-
- public PartitionBuilder setValues(List<String> values) {
- this.values = values;
- return this;
- }
-
- public PartitionBuilder addValue(String value) {
- if (values == null) values = new ArrayList<>();
- values.add(value);
- return this;
- }
-
- public PartitionBuilder setCreateTime(int createTime) {
- this.createTime = createTime;
- return this;
- }
-
- public PartitionBuilder setLastAccessTime(int lastAccessTime) {
- this.lastAccessTime = lastAccessTime;
- return this;
- }
-
- public PartitionBuilder setPartParams(Map<String, String> partParams) {
- this.partParams = partParams;
- return this;
- }
-
- public PartitionBuilder addPartParam(String key, String value) {
- if (partParams == null) partParams = new HashMap<>();
- partParams.put(key, value);
- return this;
- }
-
- public Partition build(Configuration conf) throws MetaException {
- if (tableName == null) {
- throw new MetaException("table name must be provided");
- }
- if (values == null) throw new MetaException("You must provide partition values");
- if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf);
- Partition p = new Partition(values, dbName, tableName, createTime, lastAccessTime, buildSd(),
- partParams);
- p.setCatName(catName);
- return p;
- }
-
- public Partition addToTable(IMetaStoreClient client, Configuration conf) throws TException {
- Partition p = build(conf);
- client.add_partition(p);
- return p;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java
deleted file mode 100644
index 2e7fe5a..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PrivilegeGrantInfoBuilder.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
-
-import java.io.IOException;
-
-/**
- * Builder for {@link PrivilegeGrantInfo}. The privilege is required. If not provided the grantor
- * is
- * assumed to be the current user. This is really intended for use by the
- * {@link HiveObjectPrivilegeBuilder}.
- */
-public class PrivilegeGrantInfoBuilder {
- private String privilege, grantor;
- private int createTime;
- private PrincipalType grantorType;
- private boolean grantOption;
-
- public PrivilegeGrantInfoBuilder() {
- createTime = (int)(System.currentTimeMillis() / 1000);
- grantOption = false;
- }
-
- public PrivilegeGrantInfoBuilder setPrivilege(String privilege) {
- this.privilege = privilege;
- return this;
- }
-
- public PrivilegeGrantInfoBuilder setGrantor(String grantor) {
- this.grantor = grantor;
- return this;
- }
-
- public PrivilegeGrantInfoBuilder setCreateTime(int createTime) {
- this.createTime = createTime;
- return this;
- }
-
- public PrivilegeGrantInfoBuilder setGrantorType(PrincipalType grantorType) {
- this.grantorType = grantorType;
- return this;
- }
-
- public PrivilegeGrantInfoBuilder setGrantOption(boolean grantOption) {
- this.grantOption = grantOption;
- return this;
- }
-
- public PrivilegeGrantInfo build() throws MetaException {
- if (privilege == null) {
- throw new MetaException("Privilege must be provided.");
- }
- if (grantor == null) {
- try {
- grantor = SecurityUtils.getUser();
- grantorType = PrincipalType.USER;
- } catch (IOException e) {
- throw MetaStoreUtils.newMetaException(e);
- }
- }
- return new PrivilegeGrantInfo(privilege, createTime, grantor, grantorType, grantOption);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java
deleted file mode 100644
index 0b8d189..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/RoleBuilder.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Role;
-
-/**
- * A builder for {@link Role}. The roleName and the ownerName must be provided.
- */
-public class RoleBuilder {
- private String roleName, ownerName;
- private int createTime;
-
- public RoleBuilder() {
- createTime = (int)(System.currentTimeMillis() / 1000);
- }
-
- public RoleBuilder setRoleName(String roleName) {
- this.roleName = roleName;
- return this;
- }
-
- public RoleBuilder setOwnerName(String ownerName) {
- this.ownerName = ownerName;
- return this;
- }
-
- public RoleBuilder setCreateTime(int createTime) {
- this.createTime = createTime;
- return this;
- }
-
- public Role build() throws MetaException {
- if (roleName == null || ownerName == null) {
- throw new MetaException("role name and owner name must be provided.");
- }
- return new Role(roleName, createTime, ownerName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java
deleted file mode 100644
index ec99729..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SQLCheckConstraintBuilder extends ConstraintBuilder<SQLCheckConstraintBuilder> {
- private String checkExpression;
-
- public SQLCheckConstraintBuilder() {
- super.setChild(this);
- }
-
- public SQLCheckConstraintBuilder setCheckExpression(String checkExpression) {
- this.checkExpression = checkExpression;
- return this;
- }
-
- public List<SQLCheckConstraint> build(Configuration conf) throws MetaException {
- if (checkExpression == null) {
- throw new MetaException("check expression must be set");
- }
- checkBuildable("check_constraint", conf);
- List<SQLCheckConstraint> cc = new ArrayList<>(columns.size());
- for (String column : columns) {
- cc.add(new SQLCheckConstraint(catName, dbName, tableName, column, checkExpression,
- constraintName, enable, validate, rely));
- }
- return cc;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java
deleted file mode 100644
index b24663d..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class SQLDefaultConstraintBuilder extends ConstraintBuilder<SQLDefaultConstraintBuilder> {
- private Object defaultVal;
-
- public SQLDefaultConstraintBuilder() {
- super.setChild(this);
- }
-
- public SQLDefaultConstraintBuilder setDefaultVal(Object defaultVal) {
- this.defaultVal = defaultVal;
- return this;
- }
-
- public List<SQLDefaultConstraint> build(Configuration conf) throws MetaException {
- if (defaultVal == null) {
- throw new MetaException("default value must be set");
- }
- checkBuildable("default_value", conf);
- List<SQLDefaultConstraint> dv = new ArrayList<>(columns.size());
- for (String column : columns) {
- dv.add(new SQLDefaultConstraint(catName, dbName, tableName, column,
- defaultVal.toString(), constraintName, enable, validate, rely));
- }
- return dv;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java
deleted file mode 100644
index f5adda1..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Builder for {@link SQLForeignKey}. Requires what {@link ConstraintBuilder} requires, plus
- * primary key
- * database, table, column and name.
- */
-public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder> {
- private String pkDb, pkTable, pkName;
- private List<String> pkColumns;
- private int updateRule, deleteRule;
-
- public SQLForeignKeyBuilder() {
- super.setChild(this);
- updateRule = deleteRule = 0;
- pkColumns = new ArrayList<>();
- pkDb = Warehouse.DEFAULT_DATABASE_NAME;
- }
-
- public SQLForeignKeyBuilder setPkDb(String pkDb) {
- this.pkDb = pkDb;
- return this;
- }
-
- public SQLForeignKeyBuilder setPkTable(String pkTable) {
- this.pkTable = pkTable;
- return this;
- }
-
- public SQLForeignKeyBuilder addPkColumn(String pkColumn) {
- pkColumns.add(pkColumn);
- return this;
- }
-
- public SQLForeignKeyBuilder setPkName(String pkName) {
- this.pkName = pkName;
- return this;
- }
-
- public SQLForeignKeyBuilder fromPrimaryKey(List<SQLPrimaryKey> pk) {
- pkDb = pk.get(0).getTable_db();
- pkTable = pk.get(0).getTable_name();
- for (SQLPrimaryKey pkcol : pk) pkColumns.add(pkcol.getColumn_name());
- pkName = pk.get(0).getPk_name();
- return this;
- }
-
- public SQLForeignKeyBuilder setUpdateRule(int updateRule) {
- this.updateRule = updateRule;
- return this;
- }
-
- public SQLForeignKeyBuilder setDeleteRule(int deleteRule) {
- this.deleteRule = deleteRule;
- return this;
- }
-
- public List<SQLForeignKey> build(Configuration conf) throws MetaException {
- checkBuildable("to_" + pkTable + "_foreign_key", conf);
- if (pkTable == null || pkColumns.isEmpty() || pkName == null) {
- throw new MetaException("You must provide the primary key table, columns, and name");
- }
- if (columns.size() != pkColumns.size()) {
- throw new MetaException("The number of foreign columns must match the number of primary key" +
- " columns");
- }
- List<SQLForeignKey> fk = new ArrayList<>(columns.size());
- for (int i = 0; i < columns.size(); i++) {
- SQLForeignKey keyCol = new SQLForeignKey(pkDb, pkTable, pkColumns.get(i), dbName, tableName,
- columns.get(i), getNextSeq(), updateRule, deleteRule, constraintName, pkName, enable,
- validate, rely);
- keyCol.setCatName(catName);
- fk.add(keyCol);
- }
- return fk;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java
deleted file mode 100644
index 497032e..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Builder for {@link SQLNotNullConstraint}. Only requires what {@link ConstraintBuilder} requires.
- */
-public class SQLNotNullConstraintBuilder extends ConstraintBuilder<SQLNotNullConstraintBuilder> {
-
- public SQLNotNullConstraintBuilder() {
- super.setChild(this);
- }
-
- public SQLNotNullConstraintBuilder setColName(String colName) {
- assert columns.isEmpty();
- columns.add(colName);
- return this;
- }
-
- public List<SQLNotNullConstraint> build(Configuration conf) throws MetaException {
- checkBuildable("not_null_constraint", conf);
- List<SQLNotNullConstraint> uc = new ArrayList<>(columns.size());
- for (String column : columns) {
- SQLNotNullConstraint c = new SQLNotNullConstraint(catName, dbName, tableName, columns.get(0),
- constraintName, enable, validate, rely);
- uc.add(c);
- }
- return uc;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java
deleted file mode 100644
index 40f74bd..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Builder for {@link SQLPrimaryKey}. Only requires what {@link ConstraintBuilder} requires.
- */
-public class SQLPrimaryKeyBuilder extends ConstraintBuilder<SQLPrimaryKeyBuilder> {
-
- public SQLPrimaryKeyBuilder() {
- super.setChild(this);
- }
-
- // Just to translate
- public SQLPrimaryKeyBuilder setPrimaryKeyName(String name) {
- return setConstraintName(name);
- }
-
- public List<SQLPrimaryKey> build(Configuration conf) throws MetaException {
- checkBuildable("primary_key", conf);
- List<SQLPrimaryKey> pk = new ArrayList<>(columns.size());
- for (String colName : columns) {
- SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, tableName, colName, getNextSeq(),
- constraintName, enable, validate, rely);
- keyCol.setCatName(catName);
- pk.add(keyCol);
- }
- return pk;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java
deleted file mode 100644
index 138ee15..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Builder for {@link SQLUniqueConstraint}. Only requires what {@link ConstraintBuilder} requires.
- */
-public class SQLUniqueConstraintBuilder extends ConstraintBuilder<SQLUniqueConstraintBuilder> {
-
- public SQLUniqueConstraintBuilder() {
- super.setChild(this);
- }
-
- public List<SQLUniqueConstraint> build(Configuration conf) throws MetaException {
- checkBuildable("unique_constraint", conf);
- List<SQLUniqueConstraint> uc = new ArrayList<>(columns.size());
- for (String column : columns) {
- SQLUniqueConstraint c = new SQLUniqueConstraint(catName, dbName, tableName, column, getNextSeq(),
- constraintName, enable, validate, rely);
- uc.add(c);
- }
- return uc;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java
deleted file mode 100644
index 521be3e..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.api.ISchema;
-import org.apache.hadoop.hive.metastore.api.ISchemaName;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SchemaVersion;
-import org.apache.hadoop.hive.metastore.api.SchemaVersionState;
-
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
-import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
-
-public class SchemaVersionBuilder extends SerdeAndColsBuilder<SchemaVersionBuilder> {
- private String schemaName, dbName, catName; // required
- private int version; // required
- private long createdAt; // required
- private SchemaVersionState state; // optional
- private String description; // optional
- private String schemaText; // optional
- private String fingerprint; // optional
- private String name; // optional
-
- public SchemaVersionBuilder() {
- catName = DEFAULT_CATALOG_NAME;
- dbName = DEFAULT_DATABASE_NAME;
- createdAt = System.currentTimeMillis() / 1000;
- version = -1;
- super.setChild(this);
- }
-
- public SchemaVersionBuilder setSchemaName(String schemaName) {
- this.schemaName = schemaName;
- return this;
- }
-
- public SchemaVersionBuilder setDbName(String dbName) {
- this.dbName = dbName;
- return this;
- }
-
- public SchemaVersionBuilder versionOf(ISchema schema) {
- this.catName = schema.getCatName();
- this.dbName = schema.getDbName();
- this.schemaName = schema.getName();
- return this;
- }
-
- public SchemaVersionBuilder setVersion(int version) {
- this.version = version;
- return this;
- }
-
- public SchemaVersionBuilder setCreatedAt(long createdAt) {
- this.createdAt = createdAt;
- return this;
- }
-
- public SchemaVersionBuilder setState(
- SchemaVersionState state) {
- this.state = state;
- return this;
- }
-
- public SchemaVersionBuilder setDescription(String description) {
- this.description = description;
- return this;
- }
-
- public SchemaVersionBuilder setSchemaText(String schemaText) {
- this.schemaText = schemaText;
- return this;
- }
-
- public SchemaVersionBuilder setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- return this;
- }
-
- public SchemaVersionBuilder setName(String name) {
- this.name = name;
- return this;
- }
-
- public SchemaVersion build() throws MetaException {
- if (schemaName == null || version < 0) {
- throw new MetaException("You must provide the schema name, and schema version");
- }
- SchemaVersion schemaVersion =
- new SchemaVersion(new ISchemaName(catName, dbName, schemaName), version, createdAt, getCols());
- if (state != null) schemaVersion.setState(state);
- if (description != null) schemaVersion.setDescription(description);
- if (schemaText != null) schemaVersion.setSchemaText(schemaText);
- if (fingerprint != null) schemaVersion.setFingerprint(fingerprint);
- if (name != null) schemaVersion.setName(name);
- schemaVersion.setSerDe(buildSerde());
- return schemaVersion;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SerdeAndColsBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SerdeAndColsBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SerdeAndColsBuilder.java
deleted file mode 100644
index e1405d3..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SerdeAndColsBuilder.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SerdeType;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This collects together SerdeInfo and columns, since StorageDescriptor and SchemaVersion share
- * those traits.
- * @param <T>
- */
-abstract class SerdeAndColsBuilder<T> {
- private static final String SERDE_LIB = "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
-
- private List<FieldSchema> cols;
- private String serdeName, serdeLib, serdeDescription, serdeSerializerClass, serdeDeserializerClass;
- private Map<String, String> serdeParams;
- private SerdeType serdeType;
- protected T child;
-
- protected SerdeAndColsBuilder() {
- serdeParams = new HashMap<>();
- serdeLib = SERDE_LIB;
- }
-
- protected void setChild(T child) {
- this.child = child;
- }
-
- protected SerDeInfo buildSerde() {
- SerDeInfo serDeInfo = new SerDeInfo(serdeName, serdeLib, serdeParams);
- if (serdeDescription != null) serDeInfo.setDescription(serdeDescription);
- if (serdeSerializerClass != null) serDeInfo.setSerializerClass(serdeSerializerClass);
- if (serdeDeserializerClass != null) serDeInfo.setDeserializerClass(serdeDeserializerClass);
- if (serdeType != null) serDeInfo.setSerdeType(serdeType);
- return serDeInfo;
- }
-
- protected List<FieldSchema> getCols() throws MetaException {
- if (cols == null) throw new MetaException("You must provide the columns");
- return cols;
- }
-
- public T setCols(
- List<FieldSchema> cols) {
- this.cols = cols;
- return child;
- }
-
- public T addCol(String name, String type, String comment) {
- if (cols == null) cols = new ArrayList<>();
- cols.add(new FieldSchema(name, type, comment));
- return child;
- }
-
- public T addCol(String name, String type) {
- return addCol(name, type, "");
- }
-
- public T setSerdeName(String serdeName) {
- this.serdeName = serdeName;
- return child;
- }
-
- public T setSerdeLib(String serdeLib) {
- this.serdeLib = serdeLib;
- return child;
- }
-
- public T setSerdeDescription(String serdeDescription) {
- this.serdeDescription = serdeDescription;
- return child;
- }
-
- public T setSerdeSerializerClass(String serdeSerializerClass) {
- this.serdeSerializerClass = serdeSerializerClass;
- return child;
- }
-
- public T setSerdeDeserializerClass(String serdeDeserializerClass) {
- this.serdeDeserializerClass = serdeDeserializerClass;
- return child;
- }
-
- public T setSerdeParams(
- Map<String, String> serdeParams) {
- this.serdeParams = serdeParams;
- return child;
- }
-
- public T addSerdeParam(String key, String value) {
- if (serdeParams == null) serdeParams = new HashMap<>();
- serdeParams.put(key, value);
- return child;
- }
-
- public T setSerdeType(SerdeType serdeType) {
- this.serdeType = serdeType;
- return child;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java
deleted file mode 100644
index 433e7c7..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/StorageDescriptorBuilder.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SerDeInfo;
-import org.apache.hadoop.hive.metastore.api.SkewedInfo;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Builds a {@link StorageDescriptor}. Only requires that columns be set. It picks reasonable
- * defaults for everything else. This is intended for use just by objects that have a StorageDescriptor,
- * not direct use.
- */
-abstract class StorageDescriptorBuilder<T> extends SerdeAndColsBuilder<T> {
- private static final String INPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveInputFormat";
- private static final String OUTPUT_FORMAT = "org.apache.hadoop.hive.ql.io.HiveOutputFormat";
-
- private String location, inputFormat, outputFormat;
- private int numBuckets;
- private Map<String, String> storageDescriptorParams;
- private boolean compressed, storedAsSubDirectories;
- private List<String> bucketCols, skewedColNames;
- private List<Order> sortCols;
- private List<List<String>> skewedColValues;
- private Map<List<String>, String> skewedColValueLocationMaps;
-
- protected StorageDescriptorBuilder() {
- // Set some reasonable defaults
- storageDescriptorParams = new HashMap<>();
- bucketCols = new ArrayList<>();
- sortCols = new ArrayList<>();
- numBuckets = 0;
- compressed = false;
- inputFormat = INPUT_FORMAT;
- outputFormat = OUTPUT_FORMAT;
- skewedColNames = new ArrayList<>();
- skewedColValues = new ArrayList<>();
- skewedColValueLocationMaps = new HashMap<>();
- }
-
- protected StorageDescriptor buildSd() throws MetaException {
- StorageDescriptor sd = new StorageDescriptor(getCols(), location, inputFormat, outputFormat,
- compressed, numBuckets, buildSerde(), bucketCols, sortCols, storageDescriptorParams);
- sd.setStoredAsSubDirectories(storedAsSubDirectories);
- if (skewedColNames != null) {
- SkewedInfo skewed = new SkewedInfo(skewedColNames, skewedColValues,
- skewedColValueLocationMaps);
- sd.setSkewedInfo(skewed);
- }
- return sd;
- }
-
- public T setLocation(String location) {
- this.location = location;
- return child;
- }
-
- public T setInputFormat(String inputFormat) {
- this.inputFormat = inputFormat;
- return child;
- }
-
- public T setOutputFormat(String outputFormat) {
- this.outputFormat = outputFormat;
- return child;
- }
-
- public T setNumBuckets(int numBuckets) {
- this.numBuckets = numBuckets;
- return child;
- }
-
- public T setStorageDescriptorParams(
- Map<String, String> storageDescriptorParams) {
- this.storageDescriptorParams = storageDescriptorParams;
- return child;
- }
-
- public T addStorageDescriptorParam(String key, String value) {
- if (storageDescriptorParams == null) storageDescriptorParams = new HashMap<>();
- storageDescriptorParams.put(key, value);
- return child;
- }
-
- public T setCompressed(boolean compressed) {
- this.compressed = compressed;
- return child;
- }
-
- public T setStoredAsSubDirectories(boolean storedAsSubDirectories) {
- this.storedAsSubDirectories = storedAsSubDirectories;
- return child;
- }
-
- public T setBucketCols(List<String> bucketCols) {
- this.bucketCols = bucketCols;
- return child;
- }
-
- public T addBucketCol(String bucketCol) {
- if (bucketCols == null) bucketCols = new ArrayList<>();
- bucketCols.add(bucketCol);
- return child;
- }
-
- public T setSkewedColNames(List<String> skewedColNames) {
- this.skewedColNames = skewedColNames;
- return child;
- }
-
- public T addSkewedColName(String skewedColName) {
- if (skewedColNames == null) skewedColNames = new ArrayList<>();
- skewedColNames.add(skewedColName);
- return child;
- }
-
- public T setSortCols(List<Order> sortCols) {
- this.sortCols = sortCols;
- return child;
- }
-
- public T addSortCol(String col, int order) {
- if (sortCols == null) sortCols = new ArrayList<>();
- sortCols.add(new Order(col, order));
- return child;
- }
-
- // It is not at all clear how to flatten these last two out in a useful way, and no one uses
- // these anyway.
- public T setSkewedColValues(List<List<String>> skewedColValues) {
- this.skewedColValues = skewedColValues;
- return child;
- }
-
- public T setSkewedColValueLocationMaps(
- Map<List<String>, String> skewedColValueLocationMaps) {
- this.skewedColValueLocationMaps = skewedColValueLocationMaps;
- return child;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java
deleted file mode 100644
index fed3dda..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.client.builder;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.Warehouse;
-
-import org.apache.hadoop.hive.metastore.api.CreationMetadata;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.PrincipalType;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
-import org.apache.thrift.TException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Build a {@link Table}. The database name and table name must be provided, plus whatever is
- * needed by the underlying {@link StorageDescriptorBuilder}.
- */
-public class TableBuilder extends StorageDescriptorBuilder<TableBuilder> {
- private String catName, dbName, tableName, owner, viewOriginalText, viewExpandedText, type,
- mvValidTxnList;
- private CreationMetadata cm;
- private List<FieldSchema> partCols;
- private int createTime, lastAccessTime, retention;
- private Map<String, String> tableParams;
- private boolean rewriteEnabled, temporary;
- private Set<String> mvReferencedTables;
- private PrincipalType ownerType;
-
- public TableBuilder() {
- // Set some reasonable defaults
- dbName = Warehouse.DEFAULT_DATABASE_NAME;
- tableParams = new HashMap<>();
- createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000);
- retention = 0;
- partCols = new ArrayList<>();
- type = TableType.MANAGED_TABLE.name();
- mvReferencedTables = new HashSet<>();
- temporary = false;
- super.setChild(this);
- }
-
- public TableBuilder setCatName(String catName) {
- this.catName = catName;
- return this;
- }
-
- public TableBuilder setDbName(String dbName) {
- this.dbName = dbName;
- return this;
- }
-
- public TableBuilder inDb(Database db) {
- this.dbName = db.getName();
- this.catName = db.getCatalogName();
- return this;
- }
-
- public TableBuilder setTableName(String tableName) {
- this.tableName = tableName;
- return this;
- }
-
- public TableBuilder setOwner(String owner) {
- this.owner = owner;
- return this;
- }
-
- public TableBuilder setOwnerType(PrincipalType ownerType) {
- this.ownerType = ownerType;
- return this;
- }
-
- public TableBuilder setViewOriginalText(String viewOriginalText) {
- this.viewOriginalText = viewOriginalText;
- return this;
- }
-
- public TableBuilder setViewExpandedText(String viewExpandedText) {
- this.viewExpandedText = viewExpandedText;
- return this;
- }
-
- public TableBuilder setType(String type) {
- this.type = type;
- return this;
- }
-
- public TableBuilder setCreationMetadata(CreationMetadata cm) {
- this.cm = cm;
- return this;
- }
-
- public TableBuilder setPartCols(List<FieldSchema> partCols) {
- this.partCols = partCols;
- return this;
- }
-
- public TableBuilder addPartCol(String name, String type, String comment) {
- partCols.add(new FieldSchema(name, type, comment));
- return this;
- }
-
- public TableBuilder addPartCol(String name, String type) {
- return addPartCol(name, type, "");
- }
-
- public TableBuilder setCreateTime(int createTime) {
- this.createTime = createTime;
- return this;
- }
-
- public TableBuilder setLastAccessTime(int lastAccessTime) {
- this.lastAccessTime = lastAccessTime;
- return this;
- }
-
- public TableBuilder setRetention(int retention) {
- this.retention = retention;
- return this;
- }
-
- public TableBuilder setTableParams(Map<String, String> tableParams) {
- this.tableParams = tableParams;
- return this;
- }
-
- public TableBuilder addTableParam(String key, String value) {
- if (tableParams == null) {
- tableParams = new HashMap<>();
- }
- tableParams.put(key, value);
- return this;
- }
-
- public TableBuilder setRewriteEnabled(boolean rewriteEnabled) {
- this.rewriteEnabled = rewriteEnabled;
- return this;
- }
-
- public TableBuilder setTemporary(boolean temporary) {
- this.temporary = temporary;
- return this;
- }
-
- public TableBuilder addMaterializedViewReferencedTable(String tableName) {
- mvReferencedTables.add(tableName);
- return this;
- }
-
- public TableBuilder addMaterializedViewReferencedTables(Set<String> tableNames) {
- mvReferencedTables.addAll(tableNames);
- return this;
- }
-
- public TableBuilder setMaterializedViewValidTxnList(ValidTxnList validTxnList) {
- mvValidTxnList = validTxnList.writeToString();
- return this;
- }
-
- public Table build(Configuration conf) throws MetaException {
- if (tableName == null) {
- throw new MetaException("You must set the table name");
- }
- if (ownerType == null) {
- ownerType = PrincipalType.USER;
- }
- if (owner == null) {
- try {
- owner = SecurityUtils.getUser();
- } catch (IOException e) {
- throw MetaStoreUtils.newMetaException(e);
- }
- }
- if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf);
- Table t = new Table(tableName, dbName, owner, createTime, lastAccessTime, retention, buildSd(),
- partCols, tableParams, viewOriginalText, viewExpandedText, type);
- if (rewriteEnabled) t.setRewriteEnabled(true);
- if (temporary) t.setTemporary(temporary);
- t.setCatName(catName);
- if (!mvReferencedTables.isEmpty()) {
- CreationMetadata cm = new CreationMetadata(catName, dbName, tableName, mvReferencedTables);
- if (mvValidTxnList != null) cm.setValidTxnList(mvValidTxnList);
- t.setCreationMetadata(cm);
- }
- return t;
- }
-
- public Table create(IMetaStoreClient client, Configuration conf) throws TException {
- Table t = build(conf);
- client.createTable(t);
- return t;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
deleted file mode 100644
index c18b4c7..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
-
-public class BinaryColumnStatsAggregator extends ColumnStatsAggregator {
-
- @Override
- public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo,
- List<String> partNames, boolean areAllPartsFound) throws MetaException {
- ColumnStatisticsObj statsObj = null;
- String colType = null;
- String colName = null;
- BinaryColumnStatsData aggregateData = null;
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- if (statsObj == null) {
- colName = cso.getColName();
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType,
- cso.getStatsData().getSetField());
- }
- BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats();
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- }
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- columnStatisticsData.setBinaryStats(aggregateData);
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
deleted file mode 100644
index 7630183..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
-
-public class BooleanColumnStatsAggregator extends ColumnStatsAggregator {
-
- @Override
- public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo,
- List<String> partNames, boolean areAllPartsFound) throws MetaException {
- ColumnStatisticsObj statsObj = null;
- String colType = null;
- String colName = null;
- BooleanColumnStatsData aggregateData = null;
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- if (statsObj == null) {
- colName = cso.getColName();
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType,
- cso.getStatsData().getSetField());
- }
- BooleanColumnStatsData newData = cso.getStatsData().getBooleanStats();
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- }
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- columnStatisticsData.setBooleanStats(aggregateData);
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
deleted file mode 100644
index 0beaf60..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
-
-public abstract class ColumnStatsAggregator {
- public boolean useDensityFunctionForNDVEstimation;
- public double ndvTuner;
-
- public abstract ColumnStatisticsObj aggregate(
- List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, List<String> partNames,
- boolean areAllPartsFound) throws MetaException;
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
deleted file mode 100644
index 7aaab4a..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregatorFactory.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
-
-public class ColumnStatsAggregatorFactory {
-
- private ColumnStatsAggregatorFactory() {
- }
-
- public static ColumnStatsAggregator getColumnStatsAggregator(_Fields type,
- boolean useDensityFunctionForNDVEstimation, double ndvTuner) {
- ColumnStatsAggregator agg;
- switch (type) {
- case BOOLEAN_STATS:
- agg = new BooleanColumnStatsAggregator();
- break;
- case LONG_STATS:
- agg = new LongColumnStatsAggregator();
- break;
- case DATE_STATS:
- agg = new DateColumnStatsAggregator();
- break;
- case DOUBLE_STATS:
- agg = new DoubleColumnStatsAggregator();
- break;
- case STRING_STATS:
- agg = new StringColumnStatsAggregator();
- break;
- case BINARY_STATS:
- agg = new BinaryColumnStatsAggregator();
- break;
- case DECIMAL_STATS:
- agg = new DecimalColumnStatsAggregator();
- break;
- default:
- throw new RuntimeException("Woh, bad. Unknown stats type " + type.toString());
- }
- agg.useDensityFunctionForNDVEstimation = useDensityFunctionForNDVEstimation;
- agg.ndvTuner = ndvTuner;
- return agg;
- }
-
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
- ColumnStatisticsObj cso = new ColumnStatisticsObj();
- ColumnStatisticsData csd = new ColumnStatisticsData();
- cso.setColName(colName);
- cso.setColType(colType);
- switch (type) {
- case BOOLEAN_STATS:
- csd.setBooleanStats(new BooleanColumnStatsData());
- break;
-
- case LONG_STATS:
- csd.setLongStats(new LongColumnStatsDataInspector());
- break;
-
- case DATE_STATS:
- csd.setDateStats(new DateColumnStatsDataInspector());
- break;
-
- case DOUBLE_STATS:
- csd.setDoubleStats(new DoubleColumnStatsDataInspector());
- break;
-
- case STRING_STATS:
- csd.setStringStats(new StringColumnStatsDataInspector());
- break;
-
- case BINARY_STATS:
- csd.setBinaryStats(new BinaryColumnStatsData());
- break;
-
- case DECIMAL_STATS:
- csd.setDecimalStats(new DecimalColumnStatsDataInspector());
- break;
-
- default:
- throw new RuntimeException("Woh, bad. Unknown stats type!");
- }
-
- cso.setStatsData(csd);
- return cso;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
deleted file mode 100644
index e8ff513..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.aggr;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Date;
-import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DateColumnStatsAggregator extends ColumnStatsAggregator implements
- IExtrapolatePartStatus {
-
- private static final Logger LOG = LoggerFactory.getLogger(DateColumnStatsAggregator.class);
-
- @Override
- public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo,
- List<String> partNames, boolean areAllPartsFound) throws MetaException {
- ColumnStatisticsObj statsObj = null;
- String colType = null;
- String colName = null;
- // check if all the ColumnStatisticsObjs contain stats and all the ndv are
- // bitvectors
- boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size();
- NumDistinctValueEstimator ndvEstimator = null;
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- if (statsObj == null) {
- colName = cso.getColName();
- colType = cso.getColType();
- statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType,
- cso.getStatsData().getSetField());
- LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats);
- }
- DateColumnStatsDataInspector dateColumnStats =
- (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
- if (dateColumnStats.getNdvEstimator() == null) {
- ndvEstimator = null;
- break;
- } else {
- // check if all of the bit vectors can merge
- NumDistinctValueEstimator estimator = dateColumnStats.getNdvEstimator();
- if (ndvEstimator == null) {
- ndvEstimator = estimator;
- } else {
- if (ndvEstimator.canMerge(estimator)) {
- continue;
- } else {
- ndvEstimator = null;
- break;
- }
- }
- }
- }
- if (ndvEstimator != null) {
- ndvEstimator = NumDistinctValueEstimatorFactory
- .getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null));
- ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData();
- if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) {
- DateColumnStatsDataInspector aggregateData = null;
- long lowerBound = 0;
- long higherBound = 0;
- double densityAvgSum = 0.0;
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- DateColumnStatsDataInspector newData =
- (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
- lowerBound = Math.max(lowerBound, newData.getNumDVs());
- higherBound += newData.getNumDVs();
- densityAvgSum += (diff(newData.getHighValue(), newData.getLowValue()))
- / newData.getNumDVs();
- if (ndvEstimator != null) {
- ndvEstimator.mergeEstimators(newData.getNdvEstimator());
- }
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData
- .setHighValue(max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- }
- }
- if (ndvEstimator != null) {
- // if all the ColumnStatisticsObjs contain bitvectors, we do not need to
- // use uniform distribution assumption because we can merge bitvectors
- // to get a good estimation.
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- } else {
- long estimation;
- if (useDensityFunctionForNDVEstimation) {
- // We have estimation, lowerbound and higherbound. We use estimation
- // if it is between lowerbound and higherbound.
- double densityAvg = densityAvgSum / partNames.size();
- estimation = (long) (diff(aggregateData.getHighValue(), aggregateData.getLowValue()) / densityAvg);
- if (estimation < lowerBound) {
- estimation = lowerBound;
- } else if (estimation > higherBound) {
- estimation = higherBound;
- }
- } else {
- estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner);
- }
- aggregateData.setNumDVs(estimation);
- }
- columnStatisticsData.setDateStats(aggregateData);
- } else {
- // we need extrapolation
- LOG.debug("start extrapolation for " + colName);
-
- Map<String, Integer> indexMap = new HashMap<>();
- for (int index = 0; index < partNames.size(); index++) {
- indexMap.put(partNames.get(index), index);
- }
- Map<String, Double> adjustedIndexMap = new HashMap<>();
- Map<String, ColumnStatisticsData> adjustedStatsMap = new HashMap<>();
- // while we scan the css, we also get the densityAvg, lowerbound and
- // higerbound when useDensityFunctionForNDVEstimation is true.
- double densityAvgSum = 0.0;
- if (ndvEstimator == null) {
- // if not every partition uses bitvector for ndv, we just fall back to
- // the traditional extrapolation methods.
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- String partName = csp.getPartName();
- DateColumnStatsData newData = cso.getStatsData().getDateStats();
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += diff(newData.getHighValue(), newData.getLowValue()) / newData.getNumDVs();
- }
- adjustedIndexMap.put(partName, (double) indexMap.get(partName));
- adjustedStatsMap.put(partName, cso.getStatsData());
- }
- } else {
- // we first merge all the adjacent bitvectors that we could merge and
- // derive new partition names and index.
- StringBuilder pseudoPartName = new StringBuilder();
- double pseudoIndexSum = 0;
- int length = 0;
- int curIndex = -1;
- DateColumnStatsDataInspector aggregateData = null;
- for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) {
- ColumnStatisticsObj cso = csp.getColStatsObj();
- String partName = csp.getPartName();
- DateColumnStatsDataInspector newData =
- (DateColumnStatsDataInspector) cso.getStatsData().getDateStats();
- // newData.isSetBitVectors() should be true for sure because we
- // already checked it before.
- if (indexMap.get(partName) != curIndex) {
- // There is bitvector, but it is not adjacent to the previous ones.
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDateStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += diff(aggregateData.getHighValue(), aggregateData.getLowValue())
- / aggregateData.getNumDVs();
- }
- // reset everything
- pseudoPartName = new StringBuilder();
- pseudoIndexSum = 0;
- length = 0;
- ndvEstimator = NumDistinctValueEstimatorFactory.getEmptyNumDistinctValueEstimator(ndvEstimator);
- }
- aggregateData = null;
- }
- curIndex = indexMap.get(partName);
- pseudoPartName.append(partName);
- pseudoIndexSum += curIndex;
- length++;
- curIndex++;
- if (aggregateData == null) {
- aggregateData = newData.deepCopy();
- } else {
- aggregateData.setLowValue(min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
- ndvEstimator.mergeEstimators(newData.getNdvEstimator());
- }
- if (length > 0) {
- // we have to set ndv
- adjustedIndexMap.put(pseudoPartName.toString(), pseudoIndexSum / length);
- aggregateData.setNumDVs(ndvEstimator.estimateNumDistinctValues());
- ColumnStatisticsData csd = new ColumnStatisticsData();
- csd.setDateStats(aggregateData);
- adjustedStatsMap.put(pseudoPartName.toString(), csd);
- if (useDensityFunctionForNDVEstimation) {
- densityAvgSum += diff(aggregateData.getHighValue(), aggregateData.getLowValue())
- / aggregateData.getNumDVs();
- }
- }
- }
- extrapolate(columnStatisticsData, partNames.size(), colStatsWithSourceInfo.size(),
- adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size());
- }
- LOG.debug(
- "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}",
- colName, columnStatisticsData.getDateStats().getNumDVs(), partNames.size(),
- colStatsWithSourceInfo.size());
- statsObj.setStatsData(columnStatisticsData);
- return statsObj;
- }
-
- private long diff(Date d1, Date d2) {
- return d1.getDaysSinceEpoch() - d2.getDaysSinceEpoch();
- }
-
- private Date min(Date d1, Date d2) {
- return d1.compareTo(d2) < 0 ? d1 : d2;
- }
-
- private Date max(Date d1, Date d2) {
- return d1.compareTo(d2) < 0 ? d2 : d1;
- }
-
- @Override
- public void extrapolate(ColumnStatisticsData extrapolateData, int numParts,
- int numPartsWithStats, Map<String, Double> adjustedIndexMap,
- Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) {
- int rightBorderInd = numParts;
- DateColumnStatsDataInspector extrapolateDateData = new DateColumnStatsDataInspector();
- Map<String, DateColumnStatsData> extractedAdjustedStatsMap = new HashMap<>();
- for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) {
- extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getDateStats());
- }
- List<Map.Entry<String, DateColumnStatsData>> list = new LinkedList<>(
- extractedAdjustedStatsMap.entrySet());
- // get the lowValue
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return o1.getValue().getLowValue().compareTo(o2.getValue().getLowValue());
- }
- });
- double minInd = adjustedIndexMap.get(list.get(0).getKey());
- double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long lowValue = 0;
- long min = list.get(0).getValue().getLowValue().getDaysSinceEpoch();
- long max = list.get(list.size() - 1).getValue().getLowValue().getDaysSinceEpoch();
- if (minInd == maxInd) {
- lowValue = min;
- } else if (minInd < maxInd) {
- // left border is the min
- lowValue = (long) (max - (max - min) * maxInd / (maxInd - minInd));
- } else {
- // right border is the min
- lowValue = (long) (max - (max - min) * (rightBorderInd - maxInd) / (minInd - maxInd));
- }
-
- // get the highValue
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return o1.getValue().getHighValue().compareTo(o2.getValue().getHighValue());
- }
- });
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- long highValue = 0;
- min = list.get(0).getValue().getHighValue().getDaysSinceEpoch();
- max = list.get(list.size() - 1).getValue().getHighValue().getDaysSinceEpoch();
- if (minInd == maxInd) {
- highValue = min;
- } else if (minInd < maxInd) {
- // right border is the max
- highValue = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- highValue = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
-
- // get the #nulls
- long numNulls = 0;
- for (Map.Entry<String, DateColumnStatsData> entry : extractedAdjustedStatsMap.entrySet()) {
- numNulls += entry.getValue().getNumNulls();
- }
- // we scale up sumNulls based on the number of partitions
- numNulls = numNulls * numParts / numPartsWithStats;
-
- // get the ndv
- long ndv = 0;
- Collections.sort(list, new Comparator<Map.Entry<String, DateColumnStatsData>>() {
- @Override
- public int compare(Map.Entry<String, DateColumnStatsData> o1,
- Map.Entry<String, DateColumnStatsData> o2) {
- return Long.compare(o1.getValue().getNumDVs(), o2.getValue().getNumDVs());
- }
- });
- long lowerBound = list.get(list.size() - 1).getValue().getNumDVs();
- long higherBound = 0;
- for (Map.Entry<String, DateColumnStatsData> entry : list) {
- higherBound += entry.getValue().getNumDVs();
- }
- if (useDensityFunctionForNDVEstimation && densityAvg != 0.0) {
- ndv = (long) ((highValue - lowValue) / densityAvg);
- if (ndv < lowerBound) {
- ndv = lowerBound;
- } else if (ndv > higherBound) {
- ndv = higherBound;
- }
- } else {
- minInd = adjustedIndexMap.get(list.get(0).getKey());
- maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey());
- min = list.get(0).getValue().getNumDVs();
- max = list.get(list.size() - 1).getValue().getNumDVs();
- if (minInd == maxInd) {
- ndv = min;
- } else if (minInd < maxInd) {
- // right border is the max
- ndv = (long) (min + (max - min) * (rightBorderInd - minInd) / (maxInd - minInd));
- } else {
- // left border is the max
- ndv = (long) (min + (max - min) * minInd / (minInd - maxInd));
- }
- }
- extrapolateDateData.setLowValue(new Date(lowValue));
- extrapolateDateData.setHighValue(new Date(highValue));
- extrapolateDateData.setNumNulls(numNulls);
- extrapolateDateData.setNumDVs(ndv);
- extrapolateData.setDateStats(extrapolateDateData);
- }
-}