You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2018/07/19 19:55:28 UTC
[29/51] [partial] hive git commit: HIVE-20188 : Split server-specific
code outside of standalone metastore-common (Alexander Kolbasov reviewed by
Vihang Karajgaonkar)
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java
deleted file mode 100644
index 0876410..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.columnstats.cache;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory;
-import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
-
-@SuppressWarnings("serial")
-public class StringColumnStatsDataInspector extends StringColumnStatsData {
-
- private NumDistinctValueEstimator ndvEstimator;
-
- public StringColumnStatsDataInspector() {
- super();
- }
-
- public StringColumnStatsDataInspector(long maxColLen, double avgColLen,
- long numNulls, long numDVs) {
- super(maxColLen, avgColLen, numNulls, numDVs);
- }
-
- public StringColumnStatsDataInspector(StringColumnStatsDataInspector other) {
- super(other);
- if (other.ndvEstimator != null) {
- super.setBitVectors(ndvEstimator.serialize());
- }
- }
-
- @Override
- public StringColumnStatsDataInspector deepCopy() {
- return new StringColumnStatsDataInspector(this);
- }
-
- @Override
- public byte[] getBitVectors() {
- if (ndvEstimator != null) {
- updateBitVectors();
- }
- return super.getBitVectors();
- }
-
- @Override
- public ByteBuffer bufferForBitVectors() {
- if (ndvEstimator != null) {
- updateBitVectors();
- }
- return super.bufferForBitVectors();
- }
-
- @Override
- public void setBitVectors(byte[] bitVectors) {
- super.setBitVectors(bitVectors);
- this.ndvEstimator = null;
- }
-
- @Override
- public void setBitVectors(ByteBuffer bitVectors) {
- super.setBitVectors(bitVectors);
- this.ndvEstimator = null;
- }
-
- @Override
- public void unsetBitVectors() {
- super.unsetBitVectors();
- this.ndvEstimator = null;
- }
-
- @Override
- public boolean isSetBitVectors() {
- if (ndvEstimator != null) {
- updateBitVectors();
- }
- return super.isSetBitVectors();
- }
-
- @Override
- public void setBitVectorsIsSet(boolean value) {
- if (ndvEstimator != null) {
- updateBitVectors();
- }
- super.setBitVectorsIsSet(value);
- }
-
- public NumDistinctValueEstimator getNdvEstimator() {
- if (isSetBitVectors() && getBitVectors().length != 0) {
- updateNdvEstimator();
- }
- return ndvEstimator;
- }
-
- public void setNdvEstimator(NumDistinctValueEstimator ndvEstimator) {
- super.unsetBitVectors();
- this.ndvEstimator = ndvEstimator;
- }
-
- private void updateBitVectors() {
- super.setBitVectors(ndvEstimator.serialize());
- this.ndvEstimator = null;
- }
-
- private void updateNdvEstimator() {
- this.ndvEstimator = NumDistinctValueEstimatorFactory
- .getNumDistinctValueEstimator(super.getBitVectors());
- super.unsetBitVectors();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
deleted file mode 100644
index 1c2402f..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BinaryColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
- BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
deleted file mode 100644
index fd6b87a..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-
-public class BooleanColumnStatsMerger extends ColumnStatsMerger {
-
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
- BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
- aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
- aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses());
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
deleted file mode 100644
index ce55756..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class ColumnStatsMerger {
- protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName());
-
- public abstract void merge(ColumnStatisticsObj aggregateColStats,
- ColumnStatisticsObj newColStats);
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
deleted file mode 100644
index 64d07c7..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
-import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
-
-public class ColumnStatsMergerFactory {
-
- private ColumnStatsMergerFactory() {
- }
-
- public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew,
- ColumnStatisticsObj statsObjOld) {
- ColumnStatsMerger agg;
- _Fields typeNew = statsObjNew.getStatsData().getSetField();
- _Fields typeOld = statsObjOld.getStatsData().getSetField();
- // make sure that they have the same type
- typeNew = typeNew == typeOld ? typeNew : null;
- switch (typeNew) {
- case BOOLEAN_STATS:
- agg = new BooleanColumnStatsMerger();
- break;
- case LONG_STATS: {
- agg = new LongColumnStatsMerger();
- break;
- }
- case DOUBLE_STATS: {
- agg = new DoubleColumnStatsMerger();
- break;
- }
- case STRING_STATS: {
- agg = new StringColumnStatsMerger();
- break;
- }
- case BINARY_STATS:
- agg = new BinaryColumnStatsMerger();
- break;
- case DECIMAL_STATS: {
- agg = new DecimalColumnStatsMerger();
- break;
- }
- case DATE_STATS: {
- agg = new DateColumnStatsMerger();
- break;
- }
- default:
- throw new IllegalArgumentException("Unknown stats type " + statsObjNew.getStatsData().getSetField());
- }
- return agg;
- }
-
- public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) {
- ColumnStatisticsObj cso = new ColumnStatisticsObj();
- ColumnStatisticsData csd = new ColumnStatisticsData();
- cso.setColName(colName);
- cso.setColType(colType);
- switch (type) {
- case BOOLEAN_STATS:
- csd.setBooleanStats(new BooleanColumnStatsData());
- break;
-
- case LONG_STATS:
- csd.setLongStats(new LongColumnStatsDataInspector());
- break;
-
- case DOUBLE_STATS:
- csd.setDoubleStats(new DoubleColumnStatsDataInspector());
- break;
-
- case STRING_STATS:
- csd.setStringStats(new StringColumnStatsDataInspector());
- break;
-
- case BINARY_STATS:
- csd.setBinaryStats(new BinaryColumnStatsData());
- break;
-
- case DECIMAL_STATS:
- csd.setDecimalStats(new DecimalColumnStatsDataInspector());
- break;
-
- case DATE_STATS:
- csd.setDateStats(new DateColumnStatsDataInspector());
- break;
-
- default:
- throw new IllegalArgumentException("Unknown stats type");
- }
-
- cso.setStatsData(csd);
- return cso;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
deleted file mode 100644
index 5baebbb..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Date;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
-
-public class DateColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DateColumnStatsDataInspector aggregateData =
- (DateColumnStatsDataInspector) aggregateColStats.getStatsData().getDateStats();
- DateColumnStatsDataInspector newData =
- (DateColumnStatsDataInspector) newColStats.getStatsData().getDateStats();
- Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData
- .getLowValue() : newData.getLowValue();
- aggregateData.setLowValue(lowValue);
- Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData
- .getHighValue() : newData.getHighValue();
- aggregateData.setHighValue(highValue);
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator();
- NumDistinctValueEstimator newEst = newData.getNdvEstimator();
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setNdvEstimator(oldEst);
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
deleted file mode 100644
index 517ca72..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.api.Decimal;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector;
-
-public class DecimalColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DecimalColumnStatsDataInspector aggregateData =
- (DecimalColumnStatsDataInspector) aggregateColStats.getStatsData().getDecimalStats();
- DecimalColumnStatsDataInspector newData =
- (DecimalColumnStatsDataInspector) newColStats.getStatsData().getDecimalStats();
-
- Decimal lowValue = getMin(aggregateData.getLowValue(), newData.getLowValue());
- aggregateData.setLowValue(lowValue);
-
- Decimal highValue = getMax(aggregateData.getHighValue(), newData.getHighValue());
- aggregateData.setHighValue(highValue);
-
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
-
- if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator();
- NumDistinctValueEstimator newEst = newData.getNdvEstimator();
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setNdvEstimator(oldEst);
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-
- Decimal getMax(Decimal firstValue, Decimal secondValue) {
- if (firstValue == null && secondValue == null) {
- return null;
- }
-
- if (firstValue != null && secondValue != null) {
- return firstValue.compareTo(secondValue) > 0 ? firstValue : secondValue;
- }
-
- return firstValue == null ? secondValue : firstValue;
- }
-
- Decimal getMin(Decimal firstValue, Decimal secondValue) {
- if (firstValue == null && secondValue == null) {
- return null;
- }
-
- if (firstValue != null && secondValue != null) {
- return firstValue.compareTo(secondValue) > 0 ? secondValue : firstValue;
- }
-
- return firstValue == null ? secondValue : firstValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
deleted file mode 100644
index 6a95751..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
-
-public class DoubleColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- DoubleColumnStatsDataInspector aggregateData =
- (DoubleColumnStatsDataInspector) aggregateColStats.getStatsData().getDoubleStats();
- DoubleColumnStatsDataInspector newData =
- (DoubleColumnStatsDataInspector) newColStats.getStatsData().getDoubleStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator();
- NumDistinctValueEstimator newEst = newData.getNdvEstimator();
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setNdvEstimator(oldEst);
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
deleted file mode 100644
index ca1a912..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
-
-public class LongColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- LongColumnStatsDataInspector aggregateData =
- (LongColumnStatsDataInspector) aggregateColStats.getStatsData().getLongStats();
- LongColumnStatsDataInspector newData =
- (LongColumnStatsDataInspector) newColStats.getStatsData().getLongStats();
- aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue()));
- aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator();
- NumDistinctValueEstimator newEst = newData.getNdvEstimator();
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setNdvEstimator(oldEst);
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
deleted file mode 100644
index d6b4478..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.hive.metastore.columnstats.merge;
-
-import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
-
-public class StringColumnStatsMerger extends ColumnStatsMerger {
- @Override
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- StringColumnStatsDataInspector aggregateData =
- (StringColumnStatsDataInspector) aggregateColStats.getStatsData().getStringStats();
- StringColumnStatsDataInspector newData =
- (StringColumnStatsDataInspector) newColStats.getStatsData().getStringStats();
- aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
- aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen()));
- aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
- if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) {
- aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs()));
- } else {
- NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator();
- NumDistinctValueEstimator newEst = newData.getNdvEstimator();
- long ndv = -1;
- if (oldEst.canMerge(newEst)) {
- oldEst.mergeEstimators(newEst);
- ndv = oldEst.estimateNumDistinctValues();
- aggregateData.setNdvEstimator(oldEst);
- } else {
- ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
- }
- LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
- + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
- aggregateData.setNumDVs(ndv);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
deleted file mode 100644
index f57e2ce..0000000
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.conf;
-
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Text;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.io.File;
-
-public class ConfTemplatePrinter {
-
- private Document generateTemplate() throws ParserConfigurationException {
- DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder docBuilder = dbf.newDocumentBuilder();
- Document doc = docBuilder.newDocument();
- doc.appendChild(doc.createProcessingInstruction(
- "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\""));
-
- doc.appendChild(doc.createComment("\n" +
- " Licensed to the Apache Software Foundation (ASF) under one or more\n" +
- " contributor license agreements. See the NOTICE file distributed with\n" +
- " this work for additional information regarding copyright ownership.\n" +
- " The ASF licenses this file to You under the Apache License, Version 2.0\n" +
- " (the \"License\"); you may not use this file except in compliance with\n" +
- " the License. You may obtain a copy of the License at\n" +
- "\n" +
- " http://www.apache.org/licenses/LICENSE-2.0\n" +
- "\n" +
- " Unless required by applicable law or agreed to in writing, software\n" +
- " distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
- " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
- " See the License for the specific language governing permissions and\n" +
- " limitations under the License.\n"));
-
- Element root = doc.createElement("configuration");
- doc.appendChild(root);
-
- root.appendChild(doc.createComment(
- " WARNING!!! This file is auto generated for documentation purposes ONLY! "));
- root.appendChild(doc.createComment(
- " WARNING!!! Any changes you make to this file will be ignored by the metastore. "));
- root.appendChild(doc.createComment(
- " WARNING!!! You must make your changes in metastore-site.xml instead. "));
-
- root.appendChild(doc.createComment(" Metastore Execution Parameters "));
-
- root.appendChild(doc.createComment("================================"));
- root.appendChild(doc.createComment("All time unit values have a time unit abbreviation suffix"));
- root.appendChild(doc.createComment("Any time value can take any of the units"));
- root.appendChild(doc.createComment("d = day"));
- root.appendChild(doc.createComment("h = hour"));
- root.appendChild(doc.createComment("m = minute"));
- root.appendChild(doc.createComment("s = second"));
- root.appendChild(doc.createComment("ms = millisecond"));
- root.appendChild(doc.createComment("us = microsecond"));
- root.appendChild(doc.createComment("ns = nanosecond"));
- root.appendChild(doc.createComment("================================"));
-
- for (MetastoreConf.ConfVars confVars : MetastoreConf.ConfVars.values()) {
- Element property = appendElement(root, "property", null);
- appendElement(property, "name", confVars.getVarname());
- appendElement(property, "value", confVars.getDefaultVal().toString());
- appendElement(property, "description", normalize(confVars.getDescription()));
- // wish to add new line here.
- }
- return doc;
-
- }
-
- private String normalize(String description) {
- int index = description.indexOf('\n');
- if (index < 0) {
- return description;
- }
- int prev = 0;
- StringBuilder builder = new StringBuilder(description.length() << 1);
- for (;index > 0; index = description.indexOf('\n', prev = index + 1)) {
- builder.append("\n ").append(description.substring(prev, index));
- }
- if (prev < description.length()) {
- builder.append("\n ").append(description.substring(prev));
- }
- builder.append("\n ");
- return builder.toString();
- }
-
- private void writeToFile(File template, Document document) throws TransformerException {
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
- transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
- DOMSource source = new DOMSource(document);
- StreamResult result = new StreamResult(template);
- transformer.transform(source, result);
- }
-
- private Element appendElement(Element parent, String name, String text) {
- Document document = parent.getOwnerDocument();
- Element child = document.createElement(name);
- parent.appendChild(child);
- if (text != null) {
- Text textNode = document.createTextNode(text);
- child.appendChild(textNode);
- }
- return child;
- }
-
- private void print(String fileName) throws ParserConfigurationException, TransformerException {
- Document doc = generateTemplate();
- File file = new File(fileName);
- File dir = file.getParentFile();
- // Make certain the target directory exists.
- dir.mkdirs();
- writeToFile(file, doc);
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length != 1) {
- String msg = "Usage: ConfTemplatePrinter filename";
- System.err.println(msg);
- throw new RuntimeException(msg);
- }
- ConfTemplatePrinter printer = new ConfTemplatePrinter();
- printer.print(args[0]);
- }
-}