You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@hive.apache.org by GitBox <gi...@apache.org> on 2020/04/08 02:15:00 UTC
[GitHub] [hive] jcamachor opened a new pull request #970: Hive 23100
jcamachor opened a new pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409971890
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
In the future, I hope we can get rid of that method all together and do this transformation in Calcite via rule.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409971890
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
In the future, I hope we can get rid of that method all together.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r406503082
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
##########
@@ -183,4 +185,50 @@ public static Double averageTypeSize(RelDataType type) {
}
}
+ public static double typeSize(RelDataType type, Comparable value) {
Review comment:
The method was taken from Calcite's {{RelMdSize}} as is. Interestingly, I realized that although constructor for super class is protected, I could reuse it since HiveRelMdSize is extending the class. As a bonus, I could remove `averageTypeSize` too that was there for legacy reasons and use the one in `RelMdSize`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409981518
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
+ for (int i = 1; i < children.size(); i++) {
+ T columnDesc = children.get(0);
+ T valueDesc = interpretNodeAsConstantStruct(columnDesc, children.get(i));
+ if (valueDesc == null) {
+ TypeInfo targetType = exprFactory.getTypeInfo(columnDesc);
+ if (!expressions.containsKey(targetType)) {
+ expressions.put(targetType, columnDesc);
+ }
+ T nullConst = exprFactory.createConstantExpr(targetType, null);
+ expressions.put(targetType, nullConst);
+ } else {
+ TypeInfo targetType = exprFactory.getTypeInfo(valueDesc);
+ if (!expressions.containsKey(targetType)) {
+ expressions.put(targetType, columnDesc);
+ }
+ expressions.put(targetType, valueDesc);
}
- outputOpList.add(nullConst);
}
- if (!ctx.isCBOExecuted()) {
-
- HiveConf conf;
- try {
- conf = Hive.get().getConf();
- } catch (HiveException e) {
- throw new SemanticException(e);
- }
- if (children.size() <= HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES)) {
- List<T> orOperands = exprFactory.rewriteINIntoORFuncCallExpr(children);
- if (orOperands != null) {
- if (orOperands.size() == 1) {
- orOperands.add(exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString()));
- }
- funcText = "or";
- genericUDF = new GenericUDFOPOr();
- children.clear();
- children.addAll(orOperands);
- }
+ children.clear();
+ List<T> newExprs = new ArrayList<>();
+ int numEntries = expressions.keySet().size();
+ if (numEntries == 1) {
+ children.addAll(expressions.asMap().values().iterator().next());
+ funcText = "in";
Review comment:
I am going to add more information here because I think all this part was misleading, but here is the basic idea. We split the IN clause into different IN clauses, one for each different value type. As you know, Hive and Calcite treat types in IN clauses differently and it was practically impossible to find some correct implementation with a common part unless this was done. Then I rely on Calcite to merge the IN clauses if they are type compatible. This seems to be working fine.
I think this also answers your questions below.
1) We may have multiple values per type (observe it is a multimap). This is precisely what I want, to group those values by type.
2) In previous version, when IN clause had multiple null values, a single null value was added. Now I add multiple null values (basically I omitted the deduplication). I have added the deduplication back, although this can be done by Calcite.
3) For the twisted part, the initialization needs be done within the children processing because it depends on the type of the value itself, unless it is null in which case it depends on the column type.
Could this logic be improved or rewritten? Definitely... But I can tell you I work hard on this and the problem is that there are two many corner cases and two different type systems, and it is very easy that even the most minimal change produces incorrect results in any of them. That is why I came up with this solution to split IN clause depending on value type.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408673898
##########
File path: ql/src/test/results/clientpositive/vector_case_when_2.q.out
##########
@@ -998,13 +998,13 @@ STAGE PLANS:
native: true
vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
- expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(2,1))
Review comment:
why is this decimal cast enforced?
I think this is probably because the CASE was created with a return type explicitylu set - can we let calcite deduce it's type? in case the CASE is disassembled it will be reinterpreted anyway
(can be fixed later of course)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409950147
##########
File path: ql/src/test/results/clientpositive/stat_estimate_related_col.q.out
##########
@@ -313,10 +313,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t1
- filterExpr: (((b = 2) or (b = 3)) and (b = 2)) (type: boolean)
Review comment:
It is the other way around: the OR/AND transformation does not happen anymore on the non CBO path. Note that the input query has an IN clause so it stays the same.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408683038
##########
File path: ql/src/test/results/clientpositive/perf/tez/query82.q.out
##########
@@ -85,7 +85,7 @@ Stage-0
Select Operator [SEL_86] (rows=297 width=400)
Output:["_col0","_col1","_col2","_col3"]
Filter Operator [FIL_85] (rows=297 width=404)
- predicate:(i_current_price BETWEEN 30 AND 60 and (i_manufact_id) IN (437, 129, 727, 663) and i_item_sk is not null)
Review comment:
order of conditions are changed - is that expected?
I would say evaluating a BETWEEN before an IN could be cheaper (but of course this might depend on the data in the table as well)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408706367
##########
File path: ql/src/test/org/apache/hadoop/hive/ql/parse/type/TestTypeCheckProcFactory.java
##########
@@ -78,63 +79,63 @@ public void init() {
testSubject = ExprNodeTypeCheck.getExprNodeDefaultExprProcessor();
}
- public void testOneCase(Object constValue) {
+ public void testOneCase(Object constValue) throws SemanticException {
Mockito.when(nodeDesc.getValue()).thenReturn(constValue);
Mockito.when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType);
- ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAs(typeInfo, nodeDesc);
+ ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAsConstant(typeInfo, nodeDesc);
Assert.assertNotNull(result);
Assert.assertEquals(expectedValue, result.getValue());
}
- public void testNullCase(Object constValue) {
+ public void testNullCase(Object constValue) throws SemanticException {
Mockito.when(nodeDesc.getValue()).thenReturn(constValue);
Mockito.when(typeInfo.getPrimitiveTypeEntry()).thenReturn(constType);
- ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAs(typeInfo, nodeDesc);
+ ExprNodeConstantDesc result = (ExprNodeConstantDesc) testSubject.interpretNodeAsConstant(typeInfo, nodeDesc);
Assert.assertNull(result);
}
@Test
- public void testWithSring() {
+ public void testWithSring() throws SemanticException {
testOneCase(maxValue.toString());
}
@Test
- public void testWithLSuffix() {
+ public void testWithLSuffix() throws SemanticException {
if (intType) {
testOneCase(maxValue.toString() + "L");
}
}
@Test
- public void testWithZeroFraction() {
+ public void testWithZeroFraction() throws SemanticException {
if (intType) {
testOneCase(maxValue.toString() + ".0");
}
}
@Test
- public void testWithFSuffix() {
+ public void testWithFSuffix() throws SemanticException {
testOneCase(maxValue.toString() + "f");
}
@Test
- public void testWithDSuffix() {
+ public void testWithDSuffix() throws SemanticException {
testOneCase(maxValue.toString() + "D");
}
@Test
- public void testOverflow() {
+ public void testOverflow() throws SemanticException {
if (intType) {
testNullCase(maxValue.add(BigDecimal.valueOf(1L)).toString());
}
}
@Test
- public void testWithNonZeroFraction() {
+ public void testWithNonZeroFraction() throws SemanticException {
Review comment:
nit: in case throwing an exception from a testcase: it's better to throw plain Exception - then we never have to go back there
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408767582
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java
##########
@@ -196,24 +197,11 @@ private RexNode transformIntoOrAndClause(RexBuilder rexBuilder, RexCall expressi
return expression;
}
- final List<RexNode> disjuncts = new ArrayList<>(expression.getOperands().size() - 2);
- final RexCall columnExpressions = (RexCall) expression.getOperands().get(0);
- for (int i = 1; i < expression.getOperands().size(); i++) {
- List<RexNode> conjuncts = new ArrayList<>(columnExpressions.getOperands().size() - 1);
- RexCall valueExpressions = (RexCall) expression.getOperands().get(i);
- for (int j = 0; j < columnExpressions.getOperands().size(); j++) {
- conjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- columnExpressions.getOperands().get(j),
- valueExpressions.getOperands().get(j)));
- }
- if (conjuncts.size() > 1) {
- disjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- conjuncts));
- } else {
- disjuncts.add(conjuncts.get(0));
- }
+ final List<RexNode> disjuncts = HiveCalciteUtil.transformIntoOrAndClause(
Review comment:
I believe this call should get the "expression" as argument not it's operands - that way the HiveCalciteUtil.tIOA method would have better contract - it seems like this callsite removes cases when there is a struct on the lefthandside - I believe that decision should be made by the utility method
...and then the OR will be created inside that method as well...it will be more straight forward
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408757617
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
I don't really like this method...Ibelieve this functionality should live somewhere else instead (HivePointLookupOptimizer) ?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405691900
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,112 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixUp(final RexBuilder rexBuilder,
Review comment:
Optional: Since the only thing we're fixing up is the nullability, do we want to call it fixNullability?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408770800
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/HiveFunctionHelper.java
##########
@@ -0,0 +1,549 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.Util;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToDateSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToUnixTimestampSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnixTimestampSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.GenericUDAFInfo;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Function helper for Hive.
+ */
+public class HiveFunctionHelper implements FunctionHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveFunctionHelper.class);
+
+ private final RexBuilder rexBuilder;
+ private final int maxNodesForInToOrTransformation;
+
+ public HiveFunctionHelper(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ try {
+ this.maxNodesForInToOrTransformation = HiveConf.getIntVar(
+ Hive.get().getConf(), HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES);
+ } catch (HiveException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public FunctionInfo getFunctionInfo(String functionText)
+ throws SemanticException {
+ return FunctionRegistry.getFunctionInfo(functionText);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public RelDataType getReturnType(FunctionInfo fi, List<RexNode> inputs)
+ throws SemanticException {
+ // 1) Gather inputs
+ ObjectInspector[] inputsOIs = new ObjectInspector[inputs.size()];
+ for (int i = 0; i < inputsOIs.length; i++) {
+ inputsOIs[i] = createObjectInspector(inputs.get(i));
+ }
+ // 2) Initialize and obtain return type
+ ObjectInspector oi = fi.getGenericUDF() != null ?
+ fi.getGenericUDF().initializeAndFoldConstants(inputsOIs) :
+ fi.getGenericUDTF().initialize(inputsOIs);
+ // 3) Convert to RelDataType
+ return TypeConverter.convert(
+ TypeInfoUtils.getTypeInfoFromObjectInspector(oi), rexBuilder.getTypeFactory());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<RexNode> convertInputs(FunctionInfo fi, List<RexNode> inputs,
+ RelDataType returnType)
+ throws SemanticException {
+ // 1) Obtain UDF
+ final GenericUDF genericUDF = fi.getGenericUDF();
+ final TypeInfo typeInfo = TypeConverter.convert(returnType);
+ TypeInfo targetType = null;
+
+ boolean isNumeric = genericUDF instanceof GenericUDFBaseBinary
+ && typeInfo.getCategory() == Category.PRIMITIVE
+ && PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
+ boolean isCompare = !isNumeric && genericUDF instanceof GenericUDFBaseCompare;
+ boolean isBetween = !isNumeric && genericUDF instanceof GenericUDFBetween;
+ boolean isIN = !isNumeric && genericUDF instanceof GenericUDFIn;
+
+ if (isNumeric) {
+ targetType = typeInfo;
+ } else if (genericUDF instanceof GenericUDFBaseCompare) {
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(0).getType()),
+ TypeConverter.convert(inputs.get(1).getType()));
+ } else if (genericUDF instanceof GenericUDFBetween) {
+ assert inputs.size() == 4;
+ // We skip first child as is not involved (is the revert boolean)
+ // The target type needs to account for all 3 operands
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(1).getType()),
+ FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(2).getType()),
+ TypeConverter.convert(inputs.get(3).getType())));
+ } else if (genericUDF instanceof GenericUDFIn) {
+ // We're only considering the first element of the IN list for the type
+ assert inputs.size() > 1;
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(0).getType()),
+ TypeConverter.convert(inputs.get(1).getType()));
+ }
+
+ if (targetType != null) {
+ List<RexNode> newInputs = new ArrayList<>();
+ // Convert inputs if needed
+ for (int i = 0; i < inputs.size(); ++i) {
+ RexNode input = inputs.get(i);
+ TypeInfo inputTypeInfo = TypeConverter.convert(input.getType());
+ RexNode tmpExprNode = input;
+ if (TypeInfoUtils.isConversionRequiredForComparison(targetType, inputTypeInfo)) {
+ if (isIN || isCompare) {
+ // For IN and compare, we will convert requisite children
+ tmpExprNode = convert(targetType, input);
+ } else if (isBetween) {
+ // For BETWEEN skip the first child (the revert boolean)
+ if (i > 0) {
+ tmpExprNode = convert(targetType, input);
+ }
+ } else if (isNumeric) {
+ // For numeric, we'll do minimum necessary cast - if we cast to the type
+ // of expression, bad things will happen.
+ PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(inputTypeInfo, targetType);
+ tmpExprNode = convert(minArgType, input);
+ } else {
+ throw new AssertionError("Unexpected " + targetType + " - not a numeric op or compare");
+ }
+ }
+
+ newInputs.add(tmpExprNode);
+ }
+ return newInputs;
+ }
+ return inputs;
+ }
+
+ private RexNode convert(TypeInfo targetType, RexNode input) throws SemanticException {
+ if (targetType.getCategory() == Category.PRIMITIVE) {
+ return RexNodeTypeCheck.getExprNodeDefaultExprProcessor(rexBuilder)
+ .createConversionCast(input, (PrimitiveTypeInfo) targetType);
+ } else {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) targetType; // struct
+ RexCall call = (RexCall) input;
+ List<RexNode> exprNodes = new ArrayList<>();
+ for (int j = 0; j < structTypeInfo.getAllStructFieldTypeInfos().size(); j++) {
+ exprNodes.add(
+ convert(
+ structTypeInfo.getAllStructFieldTypeInfos().get(j), call.getOperands().get(j)));
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.ROW, exprNodes);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public RexNode getExpression(String functionText, FunctionInfo fi,
+ List<RexNode> inputs, RelDataType returnType)
+ throws SemanticException {
+ // See if this is an explicit cast.
+ RexNode expr = RexNodeConverter.handleExplicitCast(
Review comment:
I think it would be probably cleaner to create a RexNodeConverter; and pass a rexBuilder to it
after that : remove the static from those methods - and the rexBuilder passed to every invocation of it...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408705436
##########
File path: ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestCounterMapping.java
##########
@@ -183,11 +183,11 @@ public void testBreakupAnd2() throws ParseException, CommandProcessorException {
List<FilterOperator> fos = pm.getAll(FilterOperator.class);
OpTreeSignature filterSig = pm.lookup(OpTreeSignature.class, fos.get(0));
Object pred = filterSig.getSig().getSigMap().get("getPredicateString");
- assertEquals("(u = 2) (type: boolean)", pred);
-
+ // TODO: Missed opportunity due to cast on constant preventing simplification.
+ // Fixed when we upgrade to Calcite 1.22.0
+ assertEquals("((u = 2) and (u) IN (1, 2, 3)) (type: boolean)", pred);
Review comment:
I believe this is in the background of all the other "interesting" IN operator usages? :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408718301
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
Review comment:
earlier: null values were not added; instead - they were omitted
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409981679
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
Review comment:
This will be addressed in https://issues.apache.org/jira/browse/HIVE-23225.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409964084
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -1050,111 +1070,130 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
childrenList.add(child);
}
}
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
} else if (ctx.isFoldExpr() && exprFactory.canConvertCASEIntoCOALESCEFuncCallExpr(genericUDF, children)) {
// Rewrite CASE into COALESCE
- desc = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), null,
+ expr = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), "coalesce",
Lists.newArrayList(children.get(0), exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString())));
if (Boolean.FALSE.equals(exprFactory.getConstantValue(children.get(1)))) {
- desc = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), null, Lists.newArrayList(desc));
+ expr = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), "not", Lists.newArrayList(expr));
}
} else {
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
}
// If the function is deterministic and the children are constants,
// we try to fold the expression to remove e.g. cast on constant
- if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(desc) &&
+ if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(expr) &&
FunctionRegistry.isConsistentWithinQuery(genericUDF) &&
exprFactory.isAllConstants(children)) {
- T constantExpr = exprFactory.foldExpr(desc);
+ T constantExpr = exprFactory.foldExpr(expr);
if (constantExpr != null) {
- desc = constantExpr;
+ expr = constantExpr;
}
}
}
- // UDFOPPositive is a no-op.
- // However, we still create it, and then remove it here, to make sure we
- // only allow
- // "+" for numeric types.
- if (exprFactory.isPOSITIVEFuncCallExpr(desc)) {
- assert (exprFactory.getExprChildren(desc).size() == 1);
- desc = exprFactory.getExprChildren(desc).get(0);
+
+ if (exprFactory.isPOSITIVEFuncCallExpr(expr)) {
+ // UDFOPPositive is a no-op.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ expr = exprFactory.getExprChildren(expr).get(0);
+ } else if (exprFactory.isNEGATIVEFuncCallExpr(expr)) {
+ // UDFOPNegative should always be folded.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ T input = exprFactory.getExprChildren(expr).get(0);
+ if (exprFactory.isConstantExpr(input)) {
Review comment:
Only if `foldExpr` was true. Now for `+` and `-` we always do it in both paths. Otherwise I observed new failures when the UDFs are initialized (this is done to get their return type). The reason is that some of these UDFs are implemented in such a way that they verify that certain input parameters are constants, and they were getting a call to expression `UDFOPNegative` on top of a literal.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409989722
##########
File path: ql/src/test/results/clientpositive/perf/tez/query82.q.out
##########
@@ -85,7 +85,7 @@ Stage-0
Select Operator [SEL_86] (rows=297 width=400)
Output:["_col0","_col1","_col2","_col3"]
Filter Operator [FIL_85] (rows=297 width=404)
- predicate:(i_current_price BETWEEN 30 AND 60 and (i_manufact_id) IN (437, 129, 727, 663) and i_item_sk is not null)
Review comment:
I checked, I thought it was worth it. `i_current_price` min is 0.0900000035762787 and max 99.9899978637695; thus, assuming uniform distribution, BETWEEN will filter ~70% of the data. On the other hand, the min for `i_manufact_id` is 1 and the max is 1000; assuming the same, IN will filter >99% of the data.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405696349
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,112 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixUp(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static RexNode fixUp(final RexBuilder rexBuilder,
+ RexNode node, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
Review comment:
equals?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409983453
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/jdbc/JDBCExpandExpressionsRule.java
##########
@@ -196,24 +197,11 @@ private RexNode transformIntoOrAndClause(RexBuilder rexBuilder, RexCall expressi
return expression;
}
- final List<RexNode> disjuncts = new ArrayList<>(expression.getOperands().size() - 2);
- final RexCall columnExpressions = (RexCall) expression.getOperands().get(0);
- for (int i = 1; i < expression.getOperands().size(); i++) {
- List<RexNode> conjuncts = new ArrayList<>(columnExpressions.getOperands().size() - 1);
- RexCall valueExpressions = (RexCall) expression.getOperands().get(i);
- for (int j = 0; j < columnExpressions.getOperands().size(); j++) {
- conjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.EQUALS,
- columnExpressions.getOperands().get(j),
- valueExpressions.getOperands().get(j)));
- }
- if (conjuncts.size() > 1) {
- disjuncts.add(rexBuilder.makeCall(
- SqlStdOperatorTable.AND,
- conjuncts));
- } else {
- disjuncts.add(conjuncts.get(0));
- }
+ final List<RexNode> disjuncts = HiveCalciteUtil.transformIntoOrAndClause(
Review comment:
Tbh that method follows same contract as all other `RexConverter` methods. I see the value on what you are suggesting, since with a better contract, those methods could be easily called from within rules, etc. Since we will address the `RexConverter` refactoring in HIVE-23227, I have moved this method to `RexConverter` and I have left an additional reminder in the JIRA:
https://issues.apache.org/jira/browse/HIVE-23227?focusedCommentId=17085421&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-17085421
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407811316
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,126 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixNullability(final RexBuilder rexBuilder,
Review comment:
A comment to add explanation as to why it might be required to explicitly adjust nullability?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408677212
##########
File path: ql/src/test/results/clientpositive/structin.q.out
##########
@@ -96,10 +96,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t11_n1
- filterExpr: (struct(id,lineid)) IN (const struct('1234-1111-0074578664','3'), const struct('1234-1111-0074578695',1)) (type: boolean)
Review comment:
how did this even worked before? :)
I guess we were lucky because IN was not understood by the reader
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409970095
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
Review comment:
I think you are right that this can be refined further. To be honest, my main goal here was just to mimic existing interface (which was based on already existing calls) and separate concepts without refactoring more than necessary... and still look at this jumbo patch. However, I think you are right. Maybe the factory should be split into smaller classes... Or maybe we should create a single method and pass information as a parameter. I have created https://issues.apache.org/jira/browse/HIVE-23225.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408764756
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java
##########
@@ -223,7 +224,13 @@ public Double visitCall(RexCall call) {
return null;
}
cost += operandCost;
- Double size = HiveRelMdSize.averageTypeSize(operand.getType());
Review comment:
I believe this is the change which have caused lot of operand orders to change
because this seems to be unrelated to HIVE-23100 to me - could it have it's own ticket?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408717117
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
Review comment:
it seems this map is keyed by types?
what guarantees that that doesn't go south if we have multiple of the same type?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408753089
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
Review comment:
all these methods are present on the interface - I'm wondering if there is some other way...
probably moving the literal related stuff into a separate thing might be beneficial - but the factory is right now templated ; so that would mean to pass something along with it to convert the constants...I don't have any good idea right now....but I'll keep thinking...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409967532
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
+ for (int i = 1; i < children.size(); i++) {
+ T columnDesc = children.get(0);
+ T valueDesc = interpretNodeAsConstantStruct(columnDesc, children.get(i));
Review comment:
Yes, you are right. The problem is that the method name is very unfortunate. This can generate struct of constants, expressions, or anything else. I changed the method name to `interpretNode` and improve the documentation for it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408724580
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
+ for (int i = 1; i < children.size(); i++) {
+ T columnDesc = children.get(0);
+ T valueDesc = interpretNodeAsConstantStruct(columnDesc, children.get(i));
+ if (valueDesc == null) {
+ TypeInfo targetType = exprFactory.getTypeInfo(columnDesc);
+ if (!expressions.containsKey(targetType)) {
Review comment:
this is twisted!
I don't think this should be neccessary...could you please move the "initialization" outside of the children processing ?
by doing I beleive you will be also able to demote the multi-map to a plain list
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408716155
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
+ for (int i = 1; i < children.size(); i++) {
+ T columnDesc = children.get(0);
+ T valueDesc = interpretNodeAsConstantStruct(columnDesc, children.get(i));
+ if (valueDesc == null) {
+ TypeInfo targetType = exprFactory.getTypeInfo(columnDesc);
+ if (!expressions.containsKey(targetType)) {
+ expressions.put(targetType, columnDesc);
+ }
+ T nullConst = exprFactory.createConstantExpr(targetType, null);
+ expressions.put(targetType, nullConst);
+ } else {
+ TypeInfo targetType = exprFactory.getTypeInfo(valueDesc);
+ if (!expressions.containsKey(targetType)) {
+ expressions.put(targetType, columnDesc);
+ }
+ expressions.put(targetType, valueDesc);
}
- outputOpList.add(nullConst);
}
- if (!ctx.isCBOExecuted()) {
-
- HiveConf conf;
- try {
- conf = Hive.get().getConf();
- } catch (HiveException e) {
- throw new SemanticException(e);
- }
- if (children.size() <= HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES)) {
- List<T> orOperands = exprFactory.rewriteINIntoORFuncCallExpr(children);
- if (orOperands != null) {
- if (orOperands.size() == 1) {
- orOperands.add(exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString()));
- }
- funcText = "or";
- genericUDF = new GenericUDFOPOr();
- children.clear();
- children.addAll(orOperands);
- }
+ children.clear();
+ List<T> newExprs = new ArrayList<>();
+ int numEntries = expressions.keySet().size();
+ if (numEntries == 1) {
+ children.addAll(expressions.asMap().values().iterator().next());
+ funcText = "in";
Review comment:
why do we use in here?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409947154
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortPredicates.java
##########
@@ -223,7 +224,13 @@ public Double visitCall(RexCall call) {
return null;
}
cost += operandCost;
- Double size = HiveRelMdSize.averageTypeSize(operand.getType());
Review comment:
Yeah... The reason why I have it in this patch is that a lot of q files were changing too without it. It seems the main reason is the subtle changes in the handling of char, varchar, and string which lead to different ordering.
I could definitely create a new issue for this, but given that it will not reduce the number of q file changes and that the method `typeValueSize` is part of Calcite, could we include it here?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409960777
##########
File path: ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query98.q.out
##########
@@ -75,7 +75,7 @@ HiveProject(i_item_desc=[$0], i_category=[$1], i_class=[$2], i_current_price=[$3
HiveAggregate(group=[{1, 2, 3, 4, 5}], agg#0=[sum($8)])
HiveJoin(condition=[=($7, $0)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(i_item_sk=[$0], i_item_id=[$1], i_item_desc=[$4], i_current_price=[$5], i_class=[$10], i_category=[$12])
- HiveFilter(condition=[IN($12, _UTF-16LE'Jewelry', _UTF-16LE'Sports', _UTF-16LE'Books')])
+ HiveFilter(condition=[IN($12, _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Sports':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")])
Review comment:
I'll tackle this as part of HIVE-23224.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408695313
##########
File path: ql/src/test/results/clientpositive/pcs.q.out
##########
@@ -1807,12 +1807,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: pcs_t1
- filterExpr: ((key = 3) or (((((ds = '2000-04-08') or key is not null) and (key = 2)) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean)
Review comment:
previously we were able to remove `key!=2` here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409973134
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
https://issues.apache.org/jira/browse/HIVE-23226
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] asfgit closed pull request #970: Hive 23100
Posted by GitBox <gi...@apache.org>.
asfgit closed pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405705933
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
##########
@@ -183,4 +185,50 @@ public static Double averageTypeSize(RelDataType type) {
}
}
+ public static double typeSize(RelDataType type, Comparable value) {
Review comment:
All the values returned seem to be of type integer. I saw the caller returns a double though. I suppose it's slightly more optimal to return a double type rather than an int type, but seems a bit weird if this is a public method that can be used by other classes.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409942255
##########
File path: ql/src/test/results/clientpositive/llap/vector_between_in.q.out
##########
@@ -269,16 +269,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_date_test
- filterExpr: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean)
+ filterExpr: ((cdecimal1) IN (2365.8945945946, -3367.6517567568) or (cdecimal1) IN (881.0135135135)) (type: boolean)
Review comment:
Yes, you are right. This is a missed merging opportunity due to different type inferred for values (although they are all decimal, note that different precision,scale inferred for them). I have created https://issues.apache.org/jira/browse/HIVE-23222 to tackle this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409949450
##########
File path: ql/src/test/results/clientpositive/structin.q.out
##########
@@ -96,10 +96,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t11_n1
- filterExpr: (struct(id,lineid)) IN (const struct('1234-1111-0074578664','3'), const struct('1234-1111-0074578695',1)) (type: boolean)
Review comment:
Yes, some of these are just hidden by Hive implementation of the operator.
Bonus point because stricter typing on the parameters made the predicate vectorizable :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408747846
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
Review comment:
there are quite a few functions here which have nothing to do with the actual implementation - they could be moved to the parent; and could be made final
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409960270
##########
File path: ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query89.q.out
##########
@@ -77,7 +77,7 @@ HiveProject(i_category=[$0], i_class=[$1], i_brand=[$2], s_store_name=[$3], s_co
HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7))])
HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
HiveProject(i_item_sk=[$0], i_brand=[$8], i_class=[$10], i_category=[$12])
- HiveFilter(condition=[AND(IN($10, _UTF-16LE'wallpaper', _UTF-16LE'parenting', _UTF-16LE'musical', _UTF-16LE'womens', _UTF-16LE'birdal', _UTF-16LE'pants'), IN($12, _UTF-16LE'Home', _UTF-16LE'Books', _UTF-16LE'Electronics', _UTF-16LE'Shoes', _UTF-16LE'Jewelry', _UTF-16LE'Men'), OR(AND(IN($12, _UTF-16LE'Home', _UTF-16LE'Books', _UTF-16LE'Electronics'), IN($10, _UTF-16LE'wallpaper', _UTF-16LE'parenting', _UTF-16LE'musical')), AND(IN($12, _UTF-16LE'Shoes', _UTF-16LE'Jewelry', _UTF-16LE'Men'), IN($10, _UTF-16LE'womens', _UTF-16LE'birdal', _UTF-16LE'pants'))))])
+ HiveFilter(condition=[AND(IN($10, _UTF-16LE'wallpaper':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'parenting':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'musical':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'womens':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'birdal':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'pants':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($12, _UTF-16LE'Home':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Electronics':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Shoes':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Men':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), OR(AND(IN($12, _UTF-16LE'Home':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Electronics':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($10, _UTF-16LE'wallpaper':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'parenting':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'musical':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")), AND(IN($12, _UTF-16LE'Shoes':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Men':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($10, _UTF-16LE'womens':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'birdal':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'pants':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"))))])
Review comment:
Yes, I agree. It does not make much sense. I have created https://issues.apache.org/jira/browse/HIVE-23224 to tackle this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407847773
##########
File path: ql/src/test/results/clientpositive/llap/vector_between_in.q.out
##########
@@ -269,16 +269,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_date_test
- filterExpr: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean)
+ filterExpr: ((cdecimal1) IN (2365.8945945946, -3367.6517567568) or (cdecimal1) IN (881.0135135135)) (type: boolean)
Review comment:
This looks like regression where expressions are not further simplified like before. May be create a follow up jira?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r406503283
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
##########
@@ -183,4 +185,50 @@ public static Double averageTypeSize(RelDataType type) {
}
}
+ public static double typeSize(RelDataType type, Comparable value) {
+ if (value == null) {
+ return 1d;
+ }
+ switch (type.getSqlTypeName()) {
+ case BOOLEAN:
+ case TINYINT:
+ return 1d;
+ case SMALLINT:
+ return 2d;
+ case INTEGER:
+ case FLOAT:
+ case REAL:
+ case DATE:
+ case TIME:
+ case TIME_WITH_LOCAL_TIME_ZONE:
+ case INTERVAL_YEAR:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_MONTH:
+ return 4d;
+ case BIGINT:
+ case DOUBLE:
+ case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_SECOND:
+ return 8d;
+ case BINARY:
+ case VARBINARY:
+ return ((ByteString) value).length();
+ case CHAR:
+ case VARCHAR:
+ return ((NlsString) value).getValue().length() * BYTES_PER_CHARACTER;
+ default:
+ return 32;
Review comment:
Marking as solved since the method is removed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409975115
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/HiveFunctionHelper.java
##########
@@ -0,0 +1,549 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.rex.RexVisitor;
+import org.apache.calcite.rex.RexVisitorImpl;
+import org.apache.calcite.sql.SqlBinaryOperator;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.Util;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRexExecutorImpl;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveExtractDate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFloorDate;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToDateSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveToUnixTimestampSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveUnixTimestampSqlOperator;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.SqlFunctionConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.GenericUDAFInfo;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
+import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Function helper for Hive.
+ */
+public class HiveFunctionHelper implements FunctionHelper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveFunctionHelper.class);
+
+ private final RexBuilder rexBuilder;
+ private final int maxNodesForInToOrTransformation;
+
+ public HiveFunctionHelper(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ try {
+ this.maxNodesForInToOrTransformation = HiveConf.getIntVar(
+ Hive.get().getConf(), HiveConf.ConfVars.HIVEOPT_TRANSFORM_IN_MAXNODES);
+ } catch (HiveException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public FunctionInfo getFunctionInfo(String functionText)
+ throws SemanticException {
+ return FunctionRegistry.getFunctionInfo(functionText);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public RelDataType getReturnType(FunctionInfo fi, List<RexNode> inputs)
+ throws SemanticException {
+ // 1) Gather inputs
+ ObjectInspector[] inputsOIs = new ObjectInspector[inputs.size()];
+ for (int i = 0; i < inputsOIs.length; i++) {
+ inputsOIs[i] = createObjectInspector(inputs.get(i));
+ }
+ // 2) Initialize and obtain return type
+ ObjectInspector oi = fi.getGenericUDF() != null ?
+ fi.getGenericUDF().initializeAndFoldConstants(inputsOIs) :
+ fi.getGenericUDTF().initialize(inputsOIs);
+ // 3) Convert to RelDataType
+ return TypeConverter.convert(
+ TypeInfoUtils.getTypeInfoFromObjectInspector(oi), rexBuilder.getTypeFactory());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<RexNode> convertInputs(FunctionInfo fi, List<RexNode> inputs,
+ RelDataType returnType)
+ throws SemanticException {
+ // 1) Obtain UDF
+ final GenericUDF genericUDF = fi.getGenericUDF();
+ final TypeInfo typeInfo = TypeConverter.convert(returnType);
+ TypeInfo targetType = null;
+
+ boolean isNumeric = genericUDF instanceof GenericUDFBaseBinary
+ && typeInfo.getCategory() == Category.PRIMITIVE
+ && PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
+ boolean isCompare = !isNumeric && genericUDF instanceof GenericUDFBaseCompare;
+ boolean isBetween = !isNumeric && genericUDF instanceof GenericUDFBetween;
+ boolean isIN = !isNumeric && genericUDF instanceof GenericUDFIn;
+
+ if (isNumeric) {
+ targetType = typeInfo;
+ } else if (genericUDF instanceof GenericUDFBaseCompare) {
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(0).getType()),
+ TypeConverter.convert(inputs.get(1).getType()));
+ } else if (genericUDF instanceof GenericUDFBetween) {
+ assert inputs.size() == 4;
+ // We skip first child as is not involved (is the revert boolean)
+ // The target type needs to account for all 3 operands
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(1).getType()),
+ FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(2).getType()),
+ TypeConverter.convert(inputs.get(3).getType())));
+ } else if (genericUDF instanceof GenericUDFIn) {
+ // We're only considering the first element of the IN list for the type
+ assert inputs.size() > 1;
+ targetType = FunctionRegistry.getCommonClassForComparison(
+ TypeConverter.convert(inputs.get(0).getType()),
+ TypeConverter.convert(inputs.get(1).getType()));
+ }
+
+ if (targetType != null) {
+ List<RexNode> newInputs = new ArrayList<>();
+ // Convert inputs if needed
+ for (int i = 0; i < inputs.size(); ++i) {
+ RexNode input = inputs.get(i);
+ TypeInfo inputTypeInfo = TypeConverter.convert(input.getType());
+ RexNode tmpExprNode = input;
+ if (TypeInfoUtils.isConversionRequiredForComparison(targetType, inputTypeInfo)) {
+ if (isIN || isCompare) {
+ // For IN and compare, we will convert requisite children
+ tmpExprNode = convert(targetType, input);
+ } else if (isBetween) {
+ // For BETWEEN skip the first child (the revert boolean)
+ if (i > 0) {
+ tmpExprNode = convert(targetType, input);
+ }
+ } else if (isNumeric) {
+ // For numeric, we'll do minimum necessary cast - if we cast to the type
+ // of expression, bad things will happen.
+ PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(inputTypeInfo, targetType);
+ tmpExprNode = convert(minArgType, input);
+ } else {
+ throw new AssertionError("Unexpected " + targetType + " - not a numeric op or compare");
+ }
+ }
+
+ newInputs.add(tmpExprNode);
+ }
+ return newInputs;
+ }
+ return inputs;
+ }
+
+ private RexNode convert(TypeInfo targetType, RexNode input) throws SemanticException {
+ if (targetType.getCategory() == Category.PRIMITIVE) {
+ return RexNodeTypeCheck.getExprNodeDefaultExprProcessor(rexBuilder)
+ .createConversionCast(input, (PrimitiveTypeInfo) targetType);
+ } else {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) targetType; // struct
+ RexCall call = (RexCall) input;
+ List<RexNode> exprNodes = new ArrayList<>();
+ for (int j = 0; j < structTypeInfo.getAllStructFieldTypeInfos().size(); j++) {
+ exprNodes.add(
+ convert(
+ structTypeInfo.getAllStructFieldTypeInfos().get(j), call.getOperands().get(j)));
+ }
+ return rexBuilder.makeCall(SqlStdOperatorTable.ROW, exprNodes);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public RexNode getExpression(String functionText, FunctionInfo fi,
+ List<RexNode> inputs, RelDataType returnType)
+ throws SemanticException {
+ // See if this is an explicit cast.
+ RexNode expr = RexNodeConverter.handleExplicitCast(
Review comment:
Tbh this was in my TODO list: many of those methods are not needed anymore in `RexConverter`. Hence my idea is to simply move them into `HiveFunctionHelper`. However, I did not want to do it as part of that patch to avoid further refactoring, as it was going to be shown as a bunch of removed code plus new code. I have created https://issues.apache.org/jira/browse/HIVE-23227 to tackle this.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408687550
##########
File path: ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query89.q.out
##########
@@ -77,7 +77,7 @@ HiveProject(i_category=[$0], i_class=[$1], i_brand=[$2], s_store_name=[$3], s_co
HiveFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($7))])
HiveTableScan(table=[[default, store_sales]], table:alias=[store_sales])
HiveProject(i_item_sk=[$0], i_brand=[$8], i_class=[$10], i_category=[$12])
- HiveFilter(condition=[AND(IN($10, _UTF-16LE'wallpaper', _UTF-16LE'parenting', _UTF-16LE'musical', _UTF-16LE'womens', _UTF-16LE'birdal', _UTF-16LE'pants'), IN($12, _UTF-16LE'Home', _UTF-16LE'Books', _UTF-16LE'Electronics', _UTF-16LE'Shoes', _UTF-16LE'Jewelry', _UTF-16LE'Men'), OR(AND(IN($12, _UTF-16LE'Home', _UTF-16LE'Books', _UTF-16LE'Electronics'), IN($10, _UTF-16LE'wallpaper', _UTF-16LE'parenting', _UTF-16LE'musical')), AND(IN($12, _UTF-16LE'Shoes', _UTF-16LE'Jewelry', _UTF-16LE'Men'), IN($10, _UTF-16LE'womens', _UTF-16LE'birdal', _UTF-16LE'pants'))))])
+ HiveFilter(condition=[AND(IN($10, _UTF-16LE'wallpaper':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'parenting':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'musical':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'womens':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'birdal':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'pants':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($12, _UTF-16LE'Home':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Electronics':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Shoes':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Men':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), OR(AND(IN($12, _UTF-16LE'Home':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Electronics':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($10, _UTF-16LE'wallpaper':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'parenting':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'musical':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")), AND(IN($12, _UTF-16LE'Shoes':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Men':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), IN($10, _UTF-16LE'womens':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'birdal':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'pants':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"))))])
Review comment:
it's hard to read what in there :)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409943454
##########
File path: ql/src/test/org/apache/hadoop/hive/ql/plan/mapping/TestCounterMapping.java
##########
@@ -183,11 +183,11 @@ public void testBreakupAnd2() throws ParseException, CommandProcessorException {
List<FilterOperator> fos = pm.getAll(FilterOperator.class);
OpTreeSignature filterSig = pm.lookup(OpTreeSignature.class, fos.get(0));
Object pred = filterSig.getSig().getSigMap().get("getPredicateString");
- assertEquals("(u = 2) (type: boolean)", pred);
-
+ // TODO: Missed opportunity due to cast on constant preventing simplification.
+ // Fixed when we upgrade to Calcite 1.22.0
+ assertEquals("((u = 2) and (u) IN (1, 2, 3)) (type: boolean)", pred);
Review comment:
Yes, when I saw this change the first time, I thought I would have to extend Calcite logic to simplify this. Then I realized it was simplifying it in 1.22. The culprit seems to be https://issues.apache.org/jira/browse/CALCITE-3525.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408678689
##########
File path: ql/src/test/results/clientpositive/stat_estimate_related_col.q.out
##########
@@ -313,10 +313,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: t1
- filterExpr: (((b = 2) or (b = 3)) and (b = 2)) (type: boolean)
Review comment:
it was intentional to not let calcite simplify this;
cbo is disabled - I'm wondering how did this IN transformation happened?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408691310
##########
File path: ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query48.q.out
##########
@@ -143,16 +143,16 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
CBO PLAN:
HiveAggregate(group=[{}], agg#0=[sum($8)])
HiveJoin(condition=[AND(=($7, $0), OR(AND($1, $9), AND($2, $10), AND($3, $11)))], joinType=[inner], algorithm=[none], cost=[not available])
- HiveProject(ca_address_sk=[$0], IN=[IN($8, _UTF-16LE'KY', _UTF-16LE'GA', _UTF-16LE'NM')], IN2=[IN($8, _UTF-16LE'MT', _UTF-16LE'OR', _UTF-16LE'IN')], IN3=[IN($8, _UTF-16LE'WI', _UTF-16LE'MO', _UTF-16LE'WV')])
- HiveFilter(condition=[AND(IN($8, _UTF-16LE'KY', _UTF-16LE'GA', _UTF-16LE'NM', _UTF-16LE'MT', _UTF-16LE'OR', _UTF-16LE'IN', _UTF-16LE'WI', _UTF-16LE'MO', _UTF-16LE'WV'), =($10, _UTF-16LE'United States'))])
+ HiveProject(ca_address_sk=[$0], IN=[IN($8, _UTF-16LE'KY':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'GA':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'NM':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")], IN2=[IN($8, _UTF-16LE'MT':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'OR':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'IN':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")], IN3=[IN($8, _UTF-16LE'WI':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'MO':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'WV':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")])
+ HiveFilter(condition=[AND(IN($8, _UTF-16LE'KY':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'GA':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'NM':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'MT':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'OR':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'IN':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'WI':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'MO':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'WV':VARCHAR(2147483647) CHARACTER SET "UTF-16LE"), =($10, _UTF-16LE'United States'))])
HiveTableScan(table=[[default, customer_address]], table:alias=[customer_address])
HiveJoin(condition=[=($0, $2)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(cd_demo_sk=[$0])
HiveFilter(condition=[AND(=($2, _UTF-16LE'M'), =($3, _UTF-16LE'4 yr Degree'))])
HiveTableScan(table=[[default, customer_demographics]], table:alias=[customer_demographics])
HiveJoin(condition=[=($0, $7)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(ss_sold_date_sk=[$0], ss_cdemo_sk=[$4], ss_addr_sk=[$6], ss_quantity=[$10], BETWEEN=[BETWEEN(false, $22, 0:DECIMAL(12, 2), 2000:DECIMAL(12, 2))], BETWEEN6=[BETWEEN(false, $22, 150:DECIMAL(12, 2), 3000:DECIMAL(12, 2))], BETWEEN7=[BETWEEN(false, $22, 50:DECIMAL(12, 2), 25000:DECIMAL(12, 2))])
- HiveFilter(condition=[AND(OR(BETWEEN(false, $13, 100:DECIMAL(3, 0), 150:DECIMAL(3, 0)), BETWEEN(false, $13, 50:DECIMAL(2, 0), 100:DECIMAL(3, 0)), BETWEEN(false, $13, 150:DECIMAL(3, 0), 200:DECIMAL(3, 0))), IS NOT NULL($4), IS NOT NULL($0), IS NOT NULL($6), IS NOT NULL($7), OR(<=(100:DECIMAL(3, 0), $13), <=($13, 150:DECIMAL(3, 0)), <=(50:DECIMAL(2, 0), $13), <=($13, 100:DECIMAL(3, 0)), <=(150:DECIMAL(3, 0), $13), <=($13, 200:DECIMAL(3, 0))), OR(<=(0:DECIMAL(12, 2), $22), <=($22, 2000:DECIMAL(12, 2)), <=(150:DECIMAL(12, 2), $22), <=($22, 3000:DECIMAL(12, 2)), <=(50:DECIMAL(12, 2), $22), <=($22, 25000:DECIMAL(12, 2))))])
+ HiveFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($0), IS NOT NULL($6), IS NOT NULL($7), OR(BETWEEN(false, $13, 100:DECIMAL(3, 0), 150:DECIMAL(3, 0)), BETWEEN(false, $13, 50:DECIMAL(2, 0), 100:DECIMAL(3, 0)), BETWEEN(false, $13, 150:DECIMAL(3, 0), 200:DECIMAL(3, 0))), OR(<=(100:DECIMAL(3, 0), $13), <=($13, 150:DECIMAL(3, 0)), <=(50:DECIMAL(2, 0), $13), <=($13, 100:DECIMAL(3, 0)), <=(150:DECIMAL(3, 0), $13), <=($13, 200:DECIMAL(3, 0))), OR(<=(0:DECIMAL(12, 2), $22), <=($22, 2000:DECIMAL(12, 2)), <=(150:DECIMAL(12, 2), $22), <=($22, 3000:DECIMAL(12, 2)), <=(50:DECIMAL(12, 2), $22), <=($22, 25000:DECIMAL(12, 2))))])
Review comment:
order of conditionals looks better :+1:
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on issue #970: Hive 23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on issue #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#issuecomment-611846017
Thanks for the feedback @scarlin-cloudera . I have pushed a new commit that addresses your comments.
Please @vineetgarg02 , @kgyrtkirk , take a look whenever you have the chance.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408680696
##########
File path: ql/src/test/results/clientpositive/spark/vector_between_in.q.out
##########
@@ -264,16 +264,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_date_test
- filterExpr: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean)
Review comment:
this rewrite is weird
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407823608
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprFactory.java
##########
@@ -61,12 +62,19 @@
/**
* Creates column expression.
*/
- protected abstract T createColumnRefExpr(ColumnInfo colInfo);
+ protected abstract T createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
Review comment:
same as above
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409971890
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
In the future, I hope we can get rid of that method all together. That is the problem of sharing same path for both factories...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r406503082
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
##########
@@ -183,4 +185,50 @@ public static Double averageTypeSize(RelDataType type) {
}
}
+ public static double typeSize(RelDataType type, Comparable value) {
Review comment:
The method was taken from Calcite's `RelMdSize` as is. Interestingly, I realized that although constructor for super class is protected, I could reuse it since HiveRelMdSize is extending the class. As a bonus, I could remove `averageTypeSize` too that was there for legacy reasons and use the one in `RelMdSize`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409971718
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/RexNodeExprFactory.java
##########
@@ -0,0 +1,1021 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.type;
+
+import com.google.common.collect.ImmutableList;
+import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSubQuery;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlCollation;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlQuantifyOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.ConversionUtil;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.NlsString;
+import org.apache.calcite.util.TimestampString;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
+import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.common.type.TimestampTZUtil;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.FunctionInfo;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException.UnsupportedFeature;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSubquerySemanticException;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRexExprList;
+import org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.type.RexNodeExprFactory.HiveNlsString.Interpretation;
+import org.apache.hadoop.hive.ql.plan.SubqueryType;
+import org.apache.hadoop.hive.ql.udf.SettableUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Expression factory for Calcite {@link RexNode}.
+ */
+public class RexNodeExprFactory extends ExprFactory<RexNode> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RexNodeExprFactory.class);
+
+ private final RexBuilder rexBuilder;
+ private final FunctionHelper functionHelper;
+
+ public RexNodeExprFactory(RexBuilder rexBuilder) {
+ this.rexBuilder = rexBuilder;
+ this.functionHelper = new HiveFunctionHelper(rexBuilder);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprInstance(Object o) {
+ return o instanceof RexNode;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ ObjectInspector inspector = colInfo.getObjectInspector();
+ if (inspector instanceof ConstantObjectInspector && inspector instanceof PrimitiveObjectInspector) {
+ return toPrimitiveConstDesc(colInfo, inspector, rexBuilder);
+ }
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ if (index < 0) {
+ throw new CalciteSemanticException("Unexpected error: Cannot find column");
+ }
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ private static RexNode toPrimitiveConstDesc(
+ ColumnInfo colInfo, ObjectInspector inspector, RexBuilder rexBuilder)
+ throws CalciteSemanticException {
+ Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
+ return rexBuilder.makeLiteral(constant,
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
+ throws CalciteSemanticException {
+ int index = rowResolver.getPosition(colInfo.getInternalName());
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index + offset);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createColumnRefExpr(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ int index = getPosition(colInfo, rowResolverList);
+ return rexBuilder.makeInputRef(
+ TypeConverter.convert(colInfo.getType(), rexBuilder.getTypeFactory()), index);
+ }
+
+ private int getPosition(ColumnInfo colInfo, List<RowResolver> rowResolverList)
+ throws SemanticException {
+ ColumnInfo tmp;
+ ColumnInfo cInfoToRet = null;
+ int position = 0;
+ for (RowResolver rr : rowResolverList) {
+ tmp = rr.get(colInfo.getTabAlias(), colInfo.getAlias());
+ if (tmp != null) {
+ if (cInfoToRet != null) {
+ throw new CalciteSemanticException("Could not resolve column name");
+ }
+ cInfoToRet = tmp;
+ position += rr.getPosition(cInfoToRet.getInternalName());
+ } else if (cInfoToRet == null) {
+ position += rr.getColumnInfos().size();
+ }
+ }
+ return position;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNullConstantExpr() {
+ return rexBuilder.makeNullLiteral(
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.NULL));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBooleanConstantExpr(String value) {
+ Boolean b = value != null ? Boolean.valueOf(value) : null;
+ return rexBuilder.makeLiteral(b,
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createBigintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Long.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createIntConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Integer.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createSmallintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Short.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.SMALLINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createTinyintConstantExpr(String value) {
+ return rexBuilder.makeLiteral(
+ new BigDecimal(Byte.valueOf(value)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TINYINT),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFloatConstantExpr(String value) {
+ Float f = Float.valueOf(value);
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Float.toString(f)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.FLOAT));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDoubleConstantExpr(String value) throws SemanticException {
+ Double d = Double.valueOf(value);
+ // TODO: The best solution is to support NaN in expression reduction.
+ if (Double.isNaN(d)) {
+ throw new CalciteSemanticException("NaN", UnsupportedFeature.Invalid_decimal);
+ }
+ return rexBuilder.makeApproxLiteral(
+ new BigDecimal(Double.toString(d)),
+ rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DOUBLE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createDecimalConstantExpr(String value, boolean allowNullValueConstantExpr) {
+ HiveDecimal hd = HiveDecimal.create(value);
+ if (!allowNullValueConstantExpr && hd == null) {
+ return null;
+ }
+ DecimalTypeInfo type = adjustType(hd);
+ return rexBuilder.makeExactLiteral(
+ hd != null ? hd.bigDecimalValue() : null,
+ TypeConverter.convert(type, rexBuilder.getTypeFactory()));
+ }
+
+ @Override
+ protected TypeInfo adjustConstantType(PrimitiveTypeInfo targetType, Object constantValue) {
+ if (constantValue instanceof HiveDecimal) {
+ return adjustType((HiveDecimal) constantValue);
+ }
+ return targetType;
+ }
+
+ private DecimalTypeInfo adjustType(HiveDecimal hd) {
+ // Note: the normalize() call with rounding in HiveDecimal will currently reduce the
+ // precision and scale of the value by throwing away trailing zeroes. This may or may
+ // not be desirable for the literals; however, this used to be the default behavior
+ // for explicit decimal literals (e.g. 1.0BD), so we keep this behavior for now.
+ int prec = 1;
+ int scale = 0;
+ if (hd != null) {
+ prec = hd.precision();
+ scale = hd.scale();
+ }
+ DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale);
+ return typeInfo;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object interpretConstantAsPrimitive(PrimitiveTypeInfo targetType, Object constantValue,
+ PrimitiveTypeInfo sourceType) {
+ // Extract string value if necessary
+ Object constantToInterpret = constantValue;
+ if (constantValue instanceof HiveNlsString) {
+ constantToInterpret = ((HiveNlsString) constantValue).getValue();
+ }
+
+ if (constantToInterpret instanceof Number || constantToInterpret instanceof String) {
+ try {
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.intTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).intValueExact();
+ } else if (PrimitiveObjectInspectorUtils.longTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).longValueExact();
+ } else if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return Double.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return Float.valueOf(constantToInterpret.toString());
+ } else if (PrimitiveObjectInspectorUtils.byteTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).byteValueExact();
+ } else if (PrimitiveObjectInspectorUtils.shortTypeEntry.equals(primitiveTypeEntry)) {
+ return toBigDecimal(constantToInterpret.toString()).shortValueExact();
+ } else if (PrimitiveObjectInspectorUtils.decimalTypeEntry.equals(primitiveTypeEntry)) {
+ HiveDecimal decimal = HiveDecimal.create(constantToInterpret.toString());
+ return decimal != null ? decimal.bigDecimalValue() : null;
+ }
+ } catch (NumberFormatException | ArithmeticException nfe) {
+ LOG.trace("Failed to narrow type of constant", nfe);
+ return null;
+ }
+ }
+
+ // Comparision of decimal and float/double happens in float/double.
+ if (constantToInterpret instanceof BigDecimal) {
+ BigDecimal bigDecimal = (BigDecimal) constantToInterpret;
+
+ PrimitiveTypeEntry primitiveTypeEntry = targetType.getPrimitiveTypeEntry();
+ if (PrimitiveObjectInspectorUtils.doubleTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.doubleValue();
+ } else if (PrimitiveObjectInspectorUtils.floatTypeEntry.equals(primitiveTypeEntry)) {
+ return bigDecimal.floatValue();
+ }
+ return bigDecimal;
+ }
+
+ String constTypeInfoName = sourceType.getTypeName();
+ if (constTypeInfoName.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME)) {
+ // because a comparison against a "string" will happen in "string" type.
+ // to avoid unintentional comparisons in "string"
+ // constants which are representing char/varchar values must be converted to the
+ // appropriate type.
+ if (targetType instanceof CharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveChar newValue = new HiveChar(constValue, length);
+ HiveChar maxCharConst = new HiveChar(constValue, HiveChar.MAX_CHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.CHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ if (targetType instanceof VarcharTypeInfo) {
+ final String constValue = constantToInterpret.toString();
+ final int length = TypeInfoUtils.getCharacterLengthForType(targetType);
+ HiveVarchar newValue = new HiveVarchar(constValue, length);
+ HiveVarchar maxCharConst = new HiveVarchar(constValue, HiveVarchar.MAX_VARCHAR_LENGTH);
+ if (maxCharConst.equals(newValue)) {
+ return makeHiveUnicodeString(Interpretation.VARCHAR, newValue.getValue());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ return constantValue;
+ }
+
+ private BigDecimal toBigDecimal(String val) {
+ if (!NumberUtils.isNumber(val)) {
+ throw new NumberFormatException("The given string is not a valid number: " + val);
+ }
+ return new BigDecimal(val.replaceAll("[dDfFlL]$", ""));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createStringConstantExpr(String value) {
+ return rexBuilder.makeCharLiteral(
+ makeHiveUnicodeString(Interpretation.STRING, value));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createDateConstantExpr(String value) {
+ Date d = Date.valueOf(value);
+ return rexBuilder.makeDateLiteral(
+ DateString.fromDaysSinceEpoch(d.toEpochDay()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampConstantExpr(String value) {
+ Timestamp t = Timestamp.valueOf(value);
+ return (RexLiteral) rexBuilder.makeLiteral(
+ TimestampString.fromMillisSinceEpoch(t.toEpochMilli()).withNanos(t.getNanos()),
+ rexBuilder.getTypeFactory().createSqlType(
+ SqlTypeName.TIMESTAMP,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP)),
+ false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createTimestampLocalTimeZoneConstantExpr(String value, ZoneId zoneId) {
+ TimestampTZ t = TimestampTZUtil.parse(value);
+
+ final TimestampString tsLocalTZString;
+ if (value == null) {
+ tsLocalTZString = null;
+ } else {
+ Instant i = t.getZonedDateTime().toInstant();
+ tsLocalTZString = TimestampString
+ .fromMillisSinceEpoch(i.toEpochMilli())
+ .withNanos(i.getNano());
+ }
+ return rexBuilder.makeTimestampWithLocalTimeZoneLiteral(
+ tsLocalTZString,
+ rexBuilder.getTypeFactory().getTypeSystem().getDefaultPrecision(SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(HiveIntervalYearMonth.valueOf(value).getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayTimeConstantExpr(String value) {
+ HiveIntervalDayTime v = HiveIntervalDayTime.valueOf(value);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalYearConstantExpr(String value) {
+ HiveIntervalYearMonth v = new HiveIntervalYearMonth(Integer.parseInt(value), 0);
+ BigDecimal totalMonths = BigDecimal.valueOf(v.getTotalMonths());
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMonthConstantExpr(String value) {
+ BigDecimal totalMonths = BigDecimal.valueOf(Integer.parseInt(value));
+ return rexBuilder.makeIntervalLiteral(totalMonths,
+ new SqlIntervalQualifier(TimeUnit.YEAR, TimeUnit.MONTH, new SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalDayConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(Integer.parseInt(value), 0, 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalHourConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, Integer.parseInt(value), 0, 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalMinuteConstantExpr(String value) {
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, Integer.parseInt(value), 0, 0);
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexLiteral createIntervalSecondConstantExpr(String value) {
+ BigDecimal bd = new BigDecimal(value);
+ BigDecimal bdSeconds = new BigDecimal(bd.toBigInteger());
+ BigDecimal bdNanos = bd.subtract(bdSeconds);
+ HiveIntervalDayTime v = new HiveIntervalDayTime(0, 0, 0, bdSeconds.intValueExact(),
+ bdNanos.multiply(NANOS_PER_SEC_BD).intValue());
+ BigDecimal secsValueBd = BigDecimal
+ .valueOf(v.getTotalSeconds() * 1000);
+ BigDecimal nanosValueBd = BigDecimal.valueOf((v).getNanos(), 6);
+ return rexBuilder.makeIntervalLiteral(secsValueBd.add(nanosValueBd),
+ new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, new
+ SqlParserPos(1, 1)));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createStructExpr(TypeInfo typeInfo, List<RexNode> operands)
+ throws CalciteSemanticException {
+ assert typeInfo instanceof StructTypeInfo;
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createConstantExpr(TypeInfo typeInfo, Object constantValue)
+ throws CalciteSemanticException {
+ if (typeInfo instanceof StructTypeInfo) {
+ List<TypeInfo> typeList = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
+ List<Object> objectList = (List<Object>) constantValue;
+ List<RexNode> operands = new ArrayList<>();
+ for (int i = 0; i < typeList.size(); i++) {
+ operands.add(
+ rexBuilder.makeLiteral(
+ objectList.get(i),
+ TypeConverter.convert(typeList.get(i), rexBuilder.getTypeFactory()),
+ false));
+ }
+ return rexBuilder.makeCall(
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()),
+ SqlStdOperatorTable.ROW,
+ operands);
+ }
+ return rexBuilder.makeLiteral(constantValue,
+ TypeConverter.convert(typeInfo, rexBuilder.getTypeFactory()), false);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createNestedColumnRefExpr(
+ TypeInfo typeInfo, RexNode expr, String fieldName, Boolean isList) throws CalciteSemanticException {
+ if (expr.getType().isStruct()) {
+ // regular case of accessing nested field in a column
+ return rexBuilder.makeFieldAccess(expr, fieldName, true);
+ } else {
+ // This may happen for schema-less tables, where columns are dynamically
+ // supplied by serdes.
+ throw new CalciteSemanticException("Unexpected rexnode : "
+ + expr.getClass().getCanonicalName(), UnsupportedFeature.Schema_less_table);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(TypeInfo returnType, GenericUDF genericUDF,
+ List<RexNode> inputs) throws SemanticException {
+ final String funcText = genericUDF.getClass().getAnnotation(Description.class).name();
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ return functionHelper.getExpression(
+ funcText, functionInfo, inputs,
+ TypeConverter.convert(returnType, rexBuilder.getTypeFactory()));
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createFuncCallExpr(GenericUDF genericUDF, String funcText,
+ List<RexNode> inputs) throws SemanticException {
+ // 1) Function resolution
+ final FunctionInfo functionInfo = functionHelper.getFunctionInfo(funcText);
+ // 2) Compute return type
+ RelDataType returnType;
+ if (genericUDF instanceof SettableUDF) {
+ returnType = TypeConverter.convert(
+ ((SettableUDF) genericUDF).getTypeInfo(), rexBuilder.getTypeFactory());
+ } else {
+ returnType = functionHelper.getReturnType(functionInfo, inputs);
+ }
+ // 3) Convert inputs (if necessary)
+ List<RexNode> newInputs = functionHelper.convertInputs(
+ functionInfo, inputs, returnType);
+ // 4) Return Calcite function
+ return functionHelper.getExpression(
+ funcText, functionInfo, newInputs, returnType);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode createExprsListExpr() {
+ return new HiveRexExprList();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void addExprToExprsList(RexNode columnList, RexNode expr) {
+ HiveRexExprList l = (HiveRexExprList) columnList;
+ l.addExpression(expr);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isConstantExpr(Object o) {
+ return o instanceof RexLiteral;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isFuncCallExpr(Object o) {
+ return o instanceof RexCall;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected Object getConstantValue(RexNode expr) {
+ if (expr.getType().getSqlTypeName() == SqlTypeName.ROW) {
+ List<Object> res = new ArrayList<>();
+ for (RexNode node : ((RexCall) expr).getOperands()) {
+ res.add(((RexLiteral) node).getValue4());
+ }
+ return res;
+ }
+ return ((RexLiteral) expr).getValue4();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getConstantValueAsString(RexNode expr) {
+ return ((RexLiteral) expr).getValueAs(String.class);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isColumnRefExpr(Object o) {
+ return o instanceof RexNode && RexUtil.isReferenceOrAccess((RexNode) o, true);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected String getColumnName(RexNode expr, RowResolver rowResolver) {
+ int index = ((RexInputRef) expr).getIndex();
+ return rowResolver.getColumnInfos().get(index).getInternalName();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isExprsListExpr(Object o) {
+ return o instanceof HiveRexExprList;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<RexNode> getExprChildren(RexNode expr) {
+ if (expr instanceof RexCall) {
+ return ((RexCall) expr).getOperands();
+ } else if (expr instanceof HiveRexExprList) {
+ return ((HiveRexExprList) expr).getExpressions();
+ }
+ return new ArrayList<>();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected TypeInfo getTypeInfo(RexNode expr) {
+ return expr.isA(SqlKind.LITERAL) ?
+ TypeConverter.convertLiteralType((RexLiteral) expr) :
+ TypeConverter.convert(expr.getType());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<TypeInfo> getStructTypeInfoList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldTypeInfos();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected List<String> getStructNameList(RexNode expr) {
+ StructTypeInfo structTypeInfo = (StructTypeInfo) TypeConverter.convert(expr.getType());
+ return structTypeInfo.getAllStructFieldNames();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isORFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.OR);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isANDFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.AND);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isPOSITIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.PLUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean isNEGATIVEFuncCallExpr(RexNode expr) {
+ return expr.isA(SqlKind.MINUS_PREFIX);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected RexNode setTypeInfo(RexNode expr, TypeInfo type) throws CalciteSemanticException {
+ RelDataType t = TypeConverter.convert(type, rexBuilder.getTypeFactory());
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ return rexBuilder.makeCall(t,
+ call.getOperator(), call.getOperands());
+ } else if (expr instanceof RexInputRef) {
+ RexInputRef inputRef = (RexInputRef) expr;
+ return rexBuilder.makeInputRef(t, inputRef.getIndex());
+ } else if (expr instanceof RexLiteral) {
+ RexLiteral literal = (RexLiteral) expr;
+ return rexBuilder.makeLiteral(RexLiteral.value(literal), t, false);
+ }
+ throw new RuntimeException("Unsupported expression type: " + expr.getClass().getCanonicalName());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected boolean canConvertCASEIntoCOALESCEFuncCallExpr(GenericUDF genericUDF, List<RexNode> inputs) {
Review comment:
I am not sure... The method is utterly ugly because it mixes UDF with Calcite RexLiteral. That logic should simply not be in `TypeCheckProcFactory`. The reason why I had to leave it there is because we convert back CASE into COALESCE while going back from Calcite to Hive. However, I think from AST to Calcite this is not even needed. Hence, I am just going to return `false` for the `RexNodeExprFactory` implementation and cross my fingers...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407823520
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/ExprFactory.java
##########
@@ -50,7 +50,8 @@
* Generates an expression from the input column. This may not necessarily
* be a column expression, e.g., if the column is a constant.
*/
- protected abstract T toExpr(ColumnInfo colInfo);
+ protected abstract T toExpr(ColumnInfo colInfo, RowResolver rowResolver, int offset)
Review comment:
Comment for `offset`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409964209
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -1050,111 +1070,130 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
childrenList.add(child);
}
}
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
} else if (ctx.isFoldExpr() && exprFactory.canConvertCASEIntoCOALESCEFuncCallExpr(genericUDF, children)) {
// Rewrite CASE into COALESCE
- desc = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), null,
+ expr = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), "coalesce",
Lists.newArrayList(children.get(0), exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString())));
if (Boolean.FALSE.equals(exprFactory.getConstantValue(children.get(1)))) {
- desc = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), null, Lists.newArrayList(desc));
+ expr = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), "not", Lists.newArrayList(expr));
}
} else {
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
}
// If the function is deterministic and the children are constants,
// we try to fold the expression to remove e.g. cast on constant
- if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(desc) &&
+ if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(expr) &&
FunctionRegistry.isConsistentWithinQuery(genericUDF) &&
exprFactory.isAllConstants(children)) {
- T constantExpr = exprFactory.foldExpr(desc);
+ T constantExpr = exprFactory.foldExpr(expr);
if (constantExpr != null) {
- desc = constantExpr;
+ expr = constantExpr;
}
}
}
- // UDFOPPositive is a no-op.
- // However, we still create it, and then remove it here, to make sure we
- // only allow
- // "+" for numeric types.
- if (exprFactory.isPOSITIVEFuncCallExpr(desc)) {
- assert (exprFactory.getExprChildren(desc).size() == 1);
- desc = exprFactory.getExprChildren(desc).get(0);
+
+ if (exprFactory.isPOSITIVEFuncCallExpr(expr)) {
+ // UDFOPPositive is a no-op.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ expr = exprFactory.getExprChildren(expr).get(0);
+ } else if (exprFactory.isNEGATIVEFuncCallExpr(expr)) {
+ // UDFOPNegative should always be folded.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ T input = exprFactory.getExprChildren(expr).get(0);
+ if (exprFactory.isConstantExpr(input)) {
Review comment:
`UDFOPPositive` is just above this block btw.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405706166
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdSize.java
##########
@@ -183,4 +185,50 @@ public static Double averageTypeSize(RelDataType type) {
}
}
+ public static double typeSize(RelDataType type, Comparable value) {
+ if (value == null) {
+ return 1d;
+ }
+ switch (type.getSqlTypeName()) {
+ case BOOLEAN:
+ case TINYINT:
+ return 1d;
+ case SMALLINT:
+ return 2d;
+ case INTEGER:
+ case FLOAT:
+ case REAL:
+ case DATE:
+ case TIME:
+ case TIME_WITH_LOCAL_TIME_ZONE:
+ case INTERVAL_YEAR:
+ case INTERVAL_YEAR_MONTH:
+ case INTERVAL_MONTH:
+ return 4d;
+ case BIGINT:
+ case DOUBLE:
+ case TIMESTAMP:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ case INTERVAL_DAY:
+ case INTERVAL_DAY_HOUR:
+ case INTERVAL_DAY_MINUTE:
+ case INTERVAL_DAY_SECOND:
+ case INTERVAL_HOUR:
+ case INTERVAL_HOUR_MINUTE:
+ case INTERVAL_HOUR_SECOND:
+ case INTERVAL_MINUTE:
+ case INTERVAL_MINUTE_SECOND:
+ case INTERVAL_SECOND:
+ return 8d;
+ case BINARY:
+ case VARBINARY:
+ return ((ByteString) value).length();
+ case CHAR:
+ case VARCHAR:
+ return ((NlsString) value).getValue().length() * BYTES_PER_CHARACTER;
+ default:
+ return 32;
Review comment:
If we do keep the class as returning double, need to change this one as well.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409949058
##########
File path: ql/src/test/results/clientpositive/vector_case_when_2.q.out
##########
@@ -998,13 +998,13 @@ STAGE PLANS:
native: true
vectorizationSchemaColumns: [0:q548284:int, 1:ROW__ID:struct<writeid:bigint,bucketid:int,rowid:bigint>]
Select Operator
- expressions: q548284 (type: int), CASE WHEN ((q548284 = 4)) THEN (0.8) WHEN ((q548284 = 5)) THEN (1) ELSE (8) END (type: decimal(2,1))
Review comment:
Good catch. It seems this is not needed indeed so we are missing an optimization opportunity here. I have created https://issues.apache.org/jira/browse/HIVE-23223.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408723055
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -972,55 +1005,42 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
}
}
if (genericUDF instanceof GenericUDFIn) {
-
- T columnDesc = children.get(0);
- List<T> outputOpList = children.subList(1, children.size());
- List<T> inOperands = new ArrayList<>(outputOpList);
- outputOpList.clear();
-
- boolean hasNullValue = false;
- for (T oldChild : inOperands) {
- if (oldChild == null) {
- hasNullValue = true;
- continue;
- }
- T newChild = interpretNodeAsStruct(columnDesc, oldChild);
- if (newChild == null) {
- hasNullValue = true;
- continue;
- }
- outputOpList.add(newChild);
- }
-
- if (hasNullValue) {
- T nullConst = exprFactory.createConstantExpr(exprFactory.getTypeInfo(columnDesc), null);
- if (outputOpList.size() == 0) {
- // we have found only null values...remove the IN ; it will be null all the time.
- return nullConst;
+ ListMultimap<TypeInfo, T> expressions = ArrayListMultimap.create();
+ for (int i = 1; i < children.size(); i++) {
+ T columnDesc = children.get(0);
+ T valueDesc = interpretNodeAsConstantStruct(columnDesc, children.get(i));
Review comment:
can't we have a column reference inside the struct?
I always thinked that we should support all kind of wierd cases...maybe we shouldn't...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407780513
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1062,6 +1060,25 @@ public RexNode apply(RelDataTypeField input) {
return HiveProject.create(input, copyInputRefs, null);
}
+ public static boolean isConstant(RexNode expr) {
Review comment:
`isLiteral` sounds more apt than `isConstant` since this is checking for literals.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407814888
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,126 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixNullability(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static RexNode fixNullability(final RexBuilder rexBuilder,
+ RexNode node, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ // This is a validation check which can become quite handy debugging type
+ // issues. Basically, we need both types to be equal, only difference should
+ // be nullability.
+ // However, we make an exception for Hive wrt CHAR type because Hive encodes
+ // the STRING type for literals within CHAR value (see {@link HiveNlsString})
+ // while Calcite always considers these literals to be a CHAR, which means
+ // that the reference may be created as a STRING or VARCHAR from AST node
+ // at parsing time but the actual type referenced is a CHAR.
+ if (refType2 == rightType) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ } else if (refType2.getFamily() == SqlTypeFamily.CHARACTER &&
+ rightType.getSqlTypeName() == SqlTypeName.CHAR && !rightType.isNullable()) {
+ return new RexInputRef(ref.getIndex(), rightType);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ }
+
+ /**
+ * The method tries to rewrite an IN function call into an OR/AND function call.
+ * For instance:
+ * <pre>
+ * (c) IN ( v1, v2, ...) => c=v1 || c=v2 || ...
+ * </pre>
+ * Or:
+ * <pre>
+ * (c,d) IN ( (v1,v2), (v3,v4), ...) => (c=v1 && d=v2) || (c=v3 && d=v4) || ...
+ * </pre>
+ *
+ * Returns null if the transformation fails, e.g., when non-deterministic
+ * calls are found in the expressions.
+ */
+ public static List<RexNode> transformIntoOrAndClause(List<RexNode> operands, RexBuilder rexBuilder) {
Review comment:
Can you add comment to explain what kind of input is expected? Without looking at the code it is not clear that the first operand is expected to be the column epxpression and rest value expressions.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405690477
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1062,6 +1060,25 @@ public RexNode apply(RelDataTypeField input) {
return HiveProject.create(input, copyInputRefs, null);
}
+ public static boolean isConstant(RexNode expr) {
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ if (call.getOperator() == SqlStdOperatorTable.ROW ||
+ call.getOperator() == SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR ||
+ call.getOperator() == SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR) {
Review comment:
Optional: Do we want to see if it's of general type SqlMultisetValueConstructor? This doesn't apply to ROW though, only to the ARRAY_VALUE and MAP_VALUE, so maybe explicit is better, although if a new MultiSetValue came to be, we'd get it for free
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409995886
##########
File path: ql/src/test/results/clientpositive/in_typecheck_char.q.out
##########
@@ -213,10 +214,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ax
- filterExpr: (((s = 'a') and (t = 'a ')) or (null and (t = 'bb '))) is null (type: boolean)
Review comment:
Indeed old predicate seems more efficient. Probably this happened through the inference of the types for IN expression. I guess we made some predicates better (the one above) while this one got worse. I will leave it in this patch but it is worth taking a look at this. I created https://issues.apache.org/jira/browse/HIVE-23229.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409993959
##########
File path: ql/src/test/results/clientpositive/pcs.q.out
##########
@@ -1807,12 +1807,12 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: pcs_t1
- filterExpr: ((key = 3) or (((((ds = '2000-04-08') or key is not null) and (key = 2)) or ((ds <> '2000-04-08') and (key = 3))) and ((key + 5) > 0))) (type: boolean)
Review comment:
Good catch. This is a missed optimization in Calcite. I am not sure why it is not happening now... I am wondering whether it is some weird interaction or maybe has to do with the typing of the constants. I have created https://issues.apache.org/jira/browse/HIVE-23228 to examine it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409975908
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
##########
@@ -936,14 +678,10 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticEx
break;
case TIMESTAMPLOCALTZ:
final TimestampString tsLocalTZString;
- if (value == null) {
- tsLocalTZString = null;
- } else {
- Instant i = ((TimestampTZ)value).getZonedDateTime().toInstant();
- tsLocalTZString = TimestampString
- .fromMillisSinceEpoch(i.toEpochMilli())
- .withNanos(i.getNano());
- }
+ Instant i = ((TimestampTZ)value).getZonedDateTime().toInstant();
Review comment:
`value` cannot be null if we are here (note the check in L569). For the rest of types, sometimes there is a check on null value indeed, but it is because value is not final and it is replaced within those case blocks.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405698053
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,112 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixUp(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static RexNode fixUp(final RexBuilder rexBuilder,
+ RexNode node, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ // This is a validation check which can become quite handy debugging type
+ // issues. Basically, we need both types to be equal, only difference should
+ // be nullability.
+ // However, we make an exception for Hive wrt CHAR type because Hive encodes
+ // the STRING type for literals within CHAR value (see {@link HiveNlsString})
+ // while Calcite always considers these literals to be a CHAR, which means
+ // that the reference may be created as a STRING or VARCHAR from AST node
+ // at parsing time but the actual type referenced is a CHAR.
+ if (refType2 == rightType) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ } else if (refType2.getFamily() == SqlTypeFamily.CHARACTER &&
+ rightType.getSqlTypeName() == SqlTypeName.CHAR && !rightType.isNullable()) {
+ return new RexInputRef(ref.getIndex(), rightType);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ }
+
+ public static List<RexNode> transformIntoOrAndClause(List<RexNode> operands, RexBuilder rexBuilder) {
Review comment:
Can you add a comment for the method?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405696536
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,112 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixUp(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static RexNode fixUp(final RexBuilder rexBuilder,
+ RexNode node, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ // This is a validation check which can become quite handy debugging type
+ // issues. Basically, we need both types to be equal, only difference should
+ // be nullability.
+ // However, we make an exception for Hive wrt CHAR type because Hive encodes
+ // the STRING type for literals within CHAR value (see {@link HiveNlsString})
+ // while Calcite always considers these literals to be a CHAR, which means
+ // that the reference may be created as a STRING or VARCHAR from AST node
+ // at parsing time but the actual type referenced is a CHAR.
+ if (refType2 == rightType) {
Review comment:
equals?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408700560
##########
File path: ql/src/test/results/clientpositive/in_typecheck_char.q.out
##########
@@ -213,10 +214,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ax
- filterExpr: (((s = 'a') and (t = 'a ')) or (null and (t = 'bb '))) is null (type: boolean)
Review comment:
the CAST will be executed at runtime for-each-row; can't we change the type of the literal?
(leave it if it's problematic)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408686018
##########
File path: ql/src/test/results/clientpositive/perf/tez/constraints/cbo_query98.q.out
##########
@@ -75,7 +75,7 @@ HiveProject(i_item_desc=[$0], i_category=[$1], i_class=[$2], i_current_price=[$3
HiveAggregate(group=[{1, 2, 3, 4, 5}], agg#0=[sum($8)])
HiveJoin(condition=[=($7, $0)], joinType=[inner], algorithm=[none], cost=[not available])
HiveProject(i_item_sk=[$0], i_item_id=[$1], i_item_desc=[$4], i_current_price=[$5], i_class=[$10], i_category=[$12])
- HiveFilter(condition=[IN($12, _UTF-16LE'Jewelry', _UTF-16LE'Sports', _UTF-16LE'Books')])
+ HiveFilter(condition=[IN($12, _UTF-16LE'Jewelry':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Sports':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", _UTF-16LE'Books':VARCHAR(2147483647) CHARACTER SET "UTF-16LE")])
Review comment:
I believe we had something earlier to remove these communicative type descriptions
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r409950319
##########
File path: ql/src/test/results/clientpositive/spark/vector_between_in.q.out
##########
@@ -264,16 +264,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: decimal_date_test
- filterExpr: (cdecimal1) IN (2365.8945945946, 881.0135135135, -3367.6517567568) (type: boolean)
Review comment:
Yes, @vineetgarg02 also noted this. I have created https://issues.apache.org/jira/browse/HIVE-23222 as a follow-up.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405706832
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
##########
@@ -227,160 +228,165 @@ public ExprNodeDesc visitCall(RexCall call) {
@Override
public ExprNodeDesc visitLiteral(RexLiteral literal) {
+ return toExprNodeConstantDesc(literal);
+ }
+
+ public static ExprNodeConstantDesc toExprNodeConstantDesc(RexLiteral literal) {
RelDataType lType = literal.getType();
if (RexLiteral.value(literal) == null) {
switch (literal.getType().getSqlTypeName()) {
- case BOOLEAN:
- return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null);
- case TINYINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, null);
- case SMALLINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.shortTypeInfo, null);
- case INTEGER:
- return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, null);
- case BIGINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, null);
- case FLOAT:
- case REAL:
- return new ExprNodeConstantDesc(TypeInfoFactory.floatTypeInfo, null);
- case DOUBLE:
- return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo, null);
- case DATE:
- return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, null);
- case TIME:
- case TIMESTAMP:
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, null);
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- HiveConf conf;
- try {
- conf = Hive.get().getConf();
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- return new ExprNodeConstantDesc(
- TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), null);
- case BINARY:
- return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, null);
- case DECIMAL:
- return new ExprNodeConstantDesc(
- TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(), lType.getScale()), null);
- case VARCHAR:
- case CHAR:
- return new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, null);
- case INTERVAL_YEAR:
- case INTERVAL_MONTH:
- case INTERVAL_YEAR_MONTH:
- return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, null);
- case INTERVAL_DAY:
- case INTERVAL_DAY_HOUR:
- case INTERVAL_DAY_MINUTE:
- case INTERVAL_DAY_SECOND:
- case INTERVAL_HOUR:
- case INTERVAL_HOUR_MINUTE:
- case INTERVAL_HOUR_SECOND:
- case INTERVAL_MINUTE:
- case INTERVAL_MINUTE_SECOND:
- case INTERVAL_SECOND:
- return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, null);
- default:
- return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, null);
+ case BOOLEAN:
Review comment:
Is this just a space difference? Hard to tell while reviewing through this editor. I know the spacing is bad here, but perhaps this fix can be done in a less intense code review (maybe file a separate jira?)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r406496073
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
##########
@@ -227,160 +228,165 @@ public ExprNodeDesc visitCall(RexCall call) {
@Override
public ExprNodeDesc visitLiteral(RexLiteral literal) {
+ return toExprNodeConstantDesc(literal);
+ }
+
+ public static ExprNodeConstantDesc toExprNodeConstantDesc(RexLiteral literal) {
RelDataType lType = literal.getType();
if (RexLiteral.value(literal) == null) {
switch (literal.getType().getSqlTypeName()) {
- case BOOLEAN:
- return new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, null);
- case TINYINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.byteTypeInfo, null);
- case SMALLINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.shortTypeInfo, null);
- case INTEGER:
- return new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, null);
- case BIGINT:
- return new ExprNodeConstantDesc(TypeInfoFactory.longTypeInfo, null);
- case FLOAT:
- case REAL:
- return new ExprNodeConstantDesc(TypeInfoFactory.floatTypeInfo, null);
- case DOUBLE:
- return new ExprNodeConstantDesc(TypeInfoFactory.doubleTypeInfo, null);
- case DATE:
- return new ExprNodeConstantDesc(TypeInfoFactory.dateTypeInfo, null);
- case TIME:
- case TIMESTAMP:
- return new ExprNodeConstantDesc(TypeInfoFactory.timestampTypeInfo, null);
- case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- HiveConf conf;
- try {
- conf = Hive.get().getConf();
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- return new ExprNodeConstantDesc(
- TypeInfoFactory.getTimestampTZTypeInfo(conf.getLocalTimeZone()), null);
- case BINARY:
- return new ExprNodeConstantDesc(TypeInfoFactory.binaryTypeInfo, null);
- case DECIMAL:
- return new ExprNodeConstantDesc(
- TypeInfoFactory.getDecimalTypeInfo(lType.getPrecision(), lType.getScale()), null);
- case VARCHAR:
- case CHAR:
- return new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, null);
- case INTERVAL_YEAR:
- case INTERVAL_MONTH:
- case INTERVAL_YEAR_MONTH:
- return new ExprNodeConstantDesc(TypeInfoFactory.intervalYearMonthTypeInfo, null);
- case INTERVAL_DAY:
- case INTERVAL_DAY_HOUR:
- case INTERVAL_DAY_MINUTE:
- case INTERVAL_DAY_SECOND:
- case INTERVAL_HOUR:
- case INTERVAL_HOUR_MINUTE:
- case INTERVAL_HOUR_SECOND:
- case INTERVAL_MINUTE:
- case INTERVAL_MINUTE_SECOND:
- case INTERVAL_SECOND:
- return new ExprNodeConstantDesc(TypeInfoFactory.intervalDayTimeTypeInfo, null);
- default:
- return new ExprNodeConstantDesc(TypeInfoFactory.voidTypeInfo, null);
+ case BOOLEAN:
Review comment:
Good catch! It was IntelliJ auto-formatting.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408774056
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
##########
@@ -936,14 +678,10 @@ protected RexNode convert(ExprNodeConstantDesc literal) throws CalciteSemanticEx
break;
case TIMESTAMPLOCALTZ:
final TimestampString tsLocalTZString;
- if (value == null) {
- tsLocalTZString = null;
- } else {
- Instant i = ((TimestampTZ)value).getZonedDateTime().toInstant();
- tsLocalTZString = TimestampString
- .fromMillisSinceEpoch(i.toEpochMilli())
- .withNanos(i.getNano());
- }
+ Instant i = ((TimestampTZ)value).getZonedDateTime().toInstant();
Review comment:
I'm not sure; but the null check seemed to be needed here.
or...there can't be a "null" which is typed as TIMESTAMPLOCALTZ?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] vineetgarg02 commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
vineetgarg02 commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r407815212
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1157,4 +1174,126 @@ public Void visitInputRef(RexInputRef inputRef) {
return inputRefSet;
}
}
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static List<RexNode> fixNullability(final RexBuilder rexBuilder,
+ List<RexNode> nodes, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(nodes);
+ }
+
+ /** Fixes up the type of all {@link RexInputRef}s in an
+ * expression to match differences in nullability.
+ *
+ * <p>Throws if there any greater inconsistencies of type. */
+ public static RexNode fixNullability(final RexBuilder rexBuilder,
+ RexNode node, final List<RelDataType> fieldTypes) {
+ return new FixNullabilityShuttle(rexBuilder, fieldTypes).apply(node);
+ }
+
+ /** Shuttle that fixes up an expression to match changes in nullability of
+ * input fields. */
+ public static class FixNullabilityShuttle extends RexShuttle {
+ private final List<RelDataType> typeList;
+ private final RexBuilder rexBuilder;
+
+ public FixNullabilityShuttle(RexBuilder rexBuilder,
+ List<RelDataType> typeList) {
+ this.typeList = typeList;
+ this.rexBuilder = rexBuilder;
+ }
+
+ @Override public RexNode visitInputRef(RexInputRef ref) {
+ final RelDataType rightType = typeList.get(ref.getIndex());
+ final RelDataType refType = ref.getType();
+ if (refType == rightType) {
+ return ref;
+ }
+ final RelDataType refType2 =
+ rexBuilder.getTypeFactory().createTypeWithNullability(refType,
+ rightType.isNullable());
+ // This is a validation check which can become quite handy debugging type
+ // issues. Basically, we need both types to be equal, only difference should
+ // be nullability.
+ // However, we make an exception for Hive wrt CHAR type because Hive encodes
+ // the STRING type for literals within CHAR value (see {@link HiveNlsString})
+ // while Calcite always considers these literals to be a CHAR, which means
+ // that the reference may be created as a STRING or VARCHAR from AST node
+ // at parsing time but the actual type referenced is a CHAR.
+ if (refType2 == rightType) {
+ return new RexInputRef(ref.getIndex(), refType2);
+ } else if (refType2.getFamily() == SqlTypeFamily.CHARACTER &&
+ rightType.getSqlTypeName() == SqlTypeName.CHAR && !rightType.isNullable()) {
+ return new RexInputRef(ref.getIndex(), rightType);
+ }
+ throw new AssertionError("mismatched type " + ref + " " + rightType);
+ }
+ }
+
+ /**
+ * The method tries to rewrite an IN function call into an OR/AND function call.
+ * For instance:
+ * <pre>
+ * (c) IN ( v1, v2, ...) => c=v1 || c=v2 || ...
+ * </pre>
+ * Or:
+ * <pre>
+ * (c,d) IN ( (v1,v2), (v3,v4), ...) => (c=v1 && d=v2) || (c=v3 && d=v4) || ...
+ * </pre>
+ *
+ * Returns null if the transformation fails, e.g., when non-deterministic
+ * calls are found in the expressions.
+ */
+ public static List<RexNode> transformIntoOrAndClause(List<RexNode> operands, RexBuilder rexBuilder) {
+ final List<RexNode> disjuncts = new ArrayList<>(operands.size() - 2);
Review comment:
Nit - This doesn't really convert IN expressions into OR but a list of expressions into OR.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] jcamachor commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
jcamachor commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r406488204
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1062,6 +1060,25 @@ public RexNode apply(RelDataTypeField input) {
return HiveProject.create(input, copyInputRefs, null);
}
+ public static boolean isConstant(RexNode expr) {
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ if (call.getOperator() == SqlStdOperatorTable.ROW ||
+ call.getOperator() == SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR ||
+ call.getOperator() == SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR) {
Review comment:
I think support to recognize `SqlMultisetValueConstructor` would need to be added in other places too in any case... We can leave it more explicit for the time being and generalize later on if needed. Fwiw this call is also very specific to Hive and how it handles the initialization of UDFs with parameters that are constant; otherwise, we would not need it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] kgyrtkirk commented on a change in pull request #970: Hive
23100
Posted by GitBox <gi...@apache.org>.
kgyrtkirk commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r408712622
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/parse/type/TypeCheckProcFactory.java
##########
@@ -1050,111 +1070,130 @@ protected T getXpathOrFuncExprNodeDesc(ASTNode expr,
childrenList.add(child);
}
}
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, childrenList);
} else if (ctx.isFoldExpr() && exprFactory.canConvertCASEIntoCOALESCEFuncCallExpr(genericUDF, children)) {
// Rewrite CASE into COALESCE
- desc = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), null,
+ expr = exprFactory.createFuncCallExpr(new GenericUDFCoalesce(), "coalesce",
Lists.newArrayList(children.get(0), exprFactory.createBooleanConstantExpr(Boolean.FALSE.toString())));
if (Boolean.FALSE.equals(exprFactory.getConstantValue(children.get(1)))) {
- desc = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), null, Lists.newArrayList(desc));
+ expr = exprFactory.createFuncCallExpr(new GenericUDFOPNot(), "not", Lists.newArrayList(expr));
}
} else {
- desc = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
+ expr = exprFactory.createFuncCallExpr(genericUDF, funcText, children);
}
// If the function is deterministic and the children are constants,
// we try to fold the expression to remove e.g. cast on constant
- if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(desc) &&
+ if (ctx.isFoldExpr() && exprFactory.isFuncCallExpr(expr) &&
FunctionRegistry.isConsistentWithinQuery(genericUDF) &&
exprFactory.isAllConstants(children)) {
- T constantExpr = exprFactory.foldExpr(desc);
+ T constantExpr = exprFactory.foldExpr(expr);
if (constantExpr != null) {
- desc = constantExpr;
+ expr = constantExpr;
}
}
}
- // UDFOPPositive is a no-op.
- // However, we still create it, and then remove it here, to make sure we
- // only allow
- // "+" for numeric types.
- if (exprFactory.isPOSITIVEFuncCallExpr(desc)) {
- assert (exprFactory.getExprChildren(desc).size() == 1);
- desc = exprFactory.getExprChildren(desc).get(0);
+
+ if (exprFactory.isPOSITIVEFuncCallExpr(expr)) {
+ // UDFOPPositive is a no-op.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ expr = exprFactory.getExprChildren(expr).get(0);
+ } else if (exprFactory.isNEGATIVEFuncCallExpr(expr)) {
+ // UDFOPNegative should always be folded.
+ assert (exprFactory.getExprChildren(expr).size() == 1);
+ T input = exprFactory.getExprChildren(expr).get(0);
+ if (exprFactory.isConstantExpr(input)) {
Review comment:
I think this should have happened around 15 lines above - or not?
are we missing the mapping of UDFOPPositive?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org
[GitHub] [hive] scarlin-cloudera commented on a change in pull request #970:
Hive 23100
Posted by GitBox <gi...@apache.org>.
scarlin-cloudera commented on a change in pull request #970: Hive 23100
URL: https://github.com/apache/hive/pull/970#discussion_r405687141
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
##########
@@ -1062,6 +1060,25 @@ public RexNode apply(RelDataTypeField input) {
return HiveProject.create(input, copyInputRefs, null);
}
+ public static boolean isConstant(RexNode expr) {
+ if (expr instanceof RexCall) {
+ RexCall call = (RexCall) expr;
+ if (call.getOperator() == SqlStdOperatorTable.ROW ||
+ call.getOperator() == SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR ||
+ call.getOperator() == SqlStdOperatorTable.MAP_VALUE_CONSTRUCTOR) {
+ // We check all operands
+ for (RexNode node : ((RexCall) expr).getOperands()) {
Review comment:
call.getOperands()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org