You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ss...@apache.org on 2016/11/02 20:08:23 UTC
[01/18] phoenix git commit: PHOENIX-3387 Hive PhoenixStorageHandler
fails with join on numeric fields
Repository: phoenix
Updated Branches:
refs/heads/4.x-HBase-0.98 1ed90b6a2 -> c1c78b2e4
refs/heads/4.x-HBase-1.1 4f0dd6a58 -> 9aa18492b
refs/heads/master 268c395e6 -> b477f370c
PHOENIX-3387 Hive PhoenixStorageHandler fails with join on numeric fields
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/86f51602
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/86f51602
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/86f51602
Branch: refs/heads/master
Commit: 86f51602b59100e67b0f87b247a07bdec6dfcb17
Parents: 268c395
Author: Sergey Soldatov <ss...@apache.org>
Authored: Thu Oct 20 23:42:39 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:36:57 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/hive/HiveTestUtil.java | 1 +
.../objectinspector/PhoenixBooleanObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixByteObjectInspector.java | 5 +++++
.../objectinspector/PhoenixDecimalObjectInspector.java | 4 ++--
.../objectinspector/PhoenixDoubleObjectInspector.java | 5 +++++
.../objectinspector/PhoenixFloatObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixIntObjectInspector.java | 11 +++++++++++
.../hive/objectinspector/PhoenixLongObjectInspector.java | 5 +++++
.../objectinspector/PhoenixShortObjectInspector.java | 7 ++++++-
9 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
index a234d24..3407ffb 100644
--- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
+++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
@@ -567,6 +567,7 @@ public class HiveTestUtil {
public void init() throws Exception {
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ conf.setBoolVar(HiveConf.ConfVars.SUBMITLOCALTASKVIACHILD, false);
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
SessionState.start(conf);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
index 0795e14..a767ca0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixBooleanObjectInspector extends AbstractPhoenixObjectInspecto
}
@Override
+ public BooleanWritable getPrimitiveWritableObject(Object o) {
+ return new BooleanWritable(get(o));
+ }
+
+ @Override
public boolean get(Object o) {
Boolean value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
index c6c5e95..a19342a 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixByteObjectInspector extends AbstractPhoenixObjectInspector<B
}
@Override
+ public ByteWritable getPrimitiveWritableObject(Object o) {
+ return new ByteWritable(get(o));
+ }
+
+ @Override
public byte get(Object o) {
Byte value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 388863a..8afe10f 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -35,7 +35,7 @@ public class PhoenixDecimalObjectInspector extends
@Override
public Object copyObject(Object o) {
- return o == null ? null : new Decimal((Decimal) o);
+ return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
}
@Override
@@ -49,7 +49,7 @@ public class PhoenixDecimalObjectInspector extends
if (o != null) {
try {
- value = new HiveDecimalWritable((HiveDecimalWritable) o);
+ value = new HiveDecimalWritable(getPrimitiveJavaObject(o));
} catch (Exception e) {
logExceptionMessage(o, "DECIMAL");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
index 25ae793..9f440ed 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixDoubleObjectInspector extends AbstractPhoenixObjectInspector
}
@Override
+ public DoubleWritable getPrimitiveWritableObject(Object o) {
+ return new DoubleWritable(get(o));
+ }
+
+ @Override
public double get(Object o) {
Double value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
index 83ad2b0..bf1badc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
@@ -38,6 +38,11 @@ public class PhoenixFloatObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public FloatWritable getPrimitiveWritableObject(Object o) {
+ return new FloatWritable(get(o));
+ }
+
+ @Override
public float get(Object o) {
Float value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
index fc9e7d0..3511ee3 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.IntWritable;
@@ -34,6 +35,16 @@ public class PhoenixIntObjectInspector extends AbstractPhoenixObjectInspector<In
}
@Override
+ public Category getCategory() {
+ return Category.PRIMITIVE;
+ }
+
+ @Override
+ public IntWritable getPrimitiveWritableObject(Object o) {
+ return new IntWritable(get(o));
+ }
+
+ @Override
public int get(Object o) {
Integer value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
index ad5cd05..554f2a4 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixLongObjectInspector extends AbstractPhoenixObjectInspector<L
}
@Override
+ public LongWritable getPrimitiveWritableObject(Object o) {
+ return new LongWritable(get(o));
+ }
+
+ @Override
public long get(Object o) {
Long value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/86f51602/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
index 1b7ec13..84529b0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
@@ -17,9 +17,9 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.ShortWritable;
public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<ShortWritable>
implements ShortObjectInspector {
@@ -34,6 +34,11 @@ public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public ShortWritable getPrimitiveWritableObject(Object o) {
+ return new ShortWritable(get(o));
+ }
+
+ @Override
public short get(Object o) {
Short value = null;
[15/18] phoenix git commit: PHOENIX-3408 arithmetic/mathematical
operations with Decimal columns failed in Hive with PheonixStorageHandler.
Posted by ss...@apache.org.
PHOENIX-3408 arithmetic/mathematical operations with Decimal columns failed in Hive with PheonixStorageHandler.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d5f73930
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d5f73930
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d5f73930
Branch: refs/heads/4.x-HBase-1.1
Commit: d5f73930a0f11f9d49b5e5b9f0a15e1fe5f58b9f
Parents: 8463d3d
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Wed Oct 26 19:26:06 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:37 2016 -0700
----------------------------------------------------------------------
.../PhoenixDecimalObjectInspector.java | 19 ++++++++++++++-----
.../PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d5f73930/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 8afe10f..3853c18 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -21,6 +21,9 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import java.math.BigDecimal;
@@ -30,17 +33,25 @@ public class PhoenixDecimalObjectInspector extends
implements HiveDecimalObjectInspector {
public PhoenixDecimalObjectInspector() {
- super(TypeInfoFactory.decimalTypeInfo);
+ this(TypeInfoFactory.decimalTypeInfo);
+ }
+
+ public PhoenixDecimalObjectInspector(PrimitiveTypeInfo typeInfo) {
+ super(typeInfo);
}
@Override
public Object copyObject(Object o) {
- return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
+ return o == null ? null : new BigDecimal(o.toString());
}
@Override
public HiveDecimal getPrimitiveJavaObject(Object o) {
- return HiveDecimal.create((BigDecimal) o);
+ if (o == null) {
+ return null;
+ }
+
+ return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create((BigDecimal) o),(DecimalTypeInfo)typeInfo);
}
@Override
@@ -56,8 +67,6 @@ public class PhoenixDecimalObjectInspector extends
}
return value;
-
-// return super.getPrimitiveWritableObject(o);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d5f73930/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 928dede..22be0fc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -111,7 +111,7 @@ public class PhoenixObjectInspectorFactory {
oi = new PhoenixTimestampObjectInspector();
break;
case DECIMAL:
- oi = new PhoenixDecimalObjectInspector();
+ oi = new PhoenixDecimalObjectInspector((PrimitiveTypeInfo) type);
break;
case BINARY:
oi = new PhoenixBinaryObjectInspector();
[17/18] phoenix git commit: PHOENIX-3422 PhoenixQueryBuilder doesn't
make value string correctly for char(/varchar) column type.
Posted by ss...@apache.org.
PHOENIX-3422 PhoenixQueryBuilder doesn't make value string correctly for char(/varchar) column type.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/973f7119
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/973f7119
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/973f7119
Branch: refs/heads/4.x-HBase-1.1
Commit: 973f711983e3dd42d15d6a2ab3920396588df2a0
Parents: f138800
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Fri Oct 28 17:13:23 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:43 2016 -0700
----------------------------------------------------------------------
phoenix-hive/pom.xml | 7 +-
.../phoenix/hive/query/PhoenixQueryBuilder.java | 129 ++++++++++---------
.../hive/util/PhoenixStorageHandlerUtil.java | 4 +-
.../hive/query/PhoenixQueryBuilderTest.java | 87 +++++++++++++
4 files changed, 163 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/973f7119/phoenix-hive/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml
index b2692f3..c01be37 100644
--- a/phoenix-hive/pom.xml
+++ b/phoenix-hive/pom.xml
@@ -110,7 +110,12 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/973f7119/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
index 8e3a972..a38814d 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
@@ -19,7 +19,9 @@ package org.apache.phoenix.hive.query;
import com.google.common.base.CharMatcher;
import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -31,12 +33,9 @@ import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.hive.util.PhoenixUtil;
+import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -662,17 +661,17 @@ public class PhoenixQueryBuilder {
comparisonOp);
if (comparisonOp.endsWith("UDFOPEqual")) { // column = 1
- appendCondition(sql, " = ", typeName, constantValues[0]);
+ sql.append(" = ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) { // column >= 1
- appendCondition(sql, " >= ", typeName, constantValues[0]);
+ sql.append(" >= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPGreaterThan")) { // column > 1
- appendCondition(sql, " > ", typeName, constantValues[0]);
+ sql.append(" > ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) { // column <= 1
- appendCondition(sql, " <= ", typeName, constantValues[0]);
+ sql.append(" <= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPLessThan")) { // column < 1
- appendCondition(sql, " < ", typeName, constantValues[0]);
+ sql.append(" < ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPNotEqual")) { // column != 1
- appendCondition(sql, " != ", typeName, constantValues[0]);
+ sql.append(" != ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("GenericUDFBetween")) {
appendBetweenCondition(jobConf, sql, condition.isNot(), typeName, constantValues);
} else if (comparisonOp.endsWith("GenericUDFIn")) {
@@ -687,44 +686,16 @@ public class PhoenixQueryBuilder {
return conditionColumnList;
}
- protected void appendCondition(StringBuilder sql, String comparisonOp, String typeName,
- String conditionValue) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(comparisonOp).append(conditionValue);
- }
- }
-
protected void appendBetweenCondition(JobConf jobConf, StringBuilder sql, boolean isNot,
String typeName, String[] conditionValues) throws
IOException {
- if (isNot) {
- sql.append(" not between ");
- } else {
- sql.append(" between ");
- }
-
try {
- Arrays.sort(PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName,
- conditionValues));
-
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValues[0]).append("'").append(" and ").append
- ("'").append(conditionValues[1]).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValues[0]).append("')").append(" and ")
- .append("to_date('").append(conditionValues[1]).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValues[0]).append("')").append(" and" +
- " ").append("to_timestamp('").append(conditionValues[1]).append("')");
- } else {
- sql.append(conditionValues[0]).append(" and ").append(conditionValues[1]);
- }
+ Object[] typedValues = PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName, conditionValues);
+ Arrays.sort(typedValues);
+
+ appendIfNot(isNot, sql).append(" between ")
+ .append(Joiner.on(" and ").join(createConstantString(typeName, typedValues[0]),
+ createConstantString(typeName, typedValues[1])));
} catch (Exception e) {
throw new IOException(e);
}
@@ -732,29 +703,63 @@ public class PhoenixQueryBuilder {
protected void appendInCondition(StringBuilder sql, boolean isNot, String typeName, String[]
conditionValues) {
- if (isNot) {
- sql.append(" not in (");
- } else {
- sql.append(" in (");
+ List<Object> wrappedConstants = Lists.newArrayListWithCapacity(conditionValues.length);
+ for (String conditionValue : conditionValues) {
+ wrappedConstants.add(createConstantString(typeName, conditionValue));
}
- for (String conditionValue : conditionValues) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(conditionValue);
- }
+ appendIfNot(isNot, sql)
+ .append(" in (")
+ .append(Joiner.on(", ").join(wrappedConstants))
+ .append(")");
+ }
- sql.append(", ");
+ private StringBuilder appendIfNot(boolean isNot, StringBuilder sb) {
+ return isNot ? sb.append(" not") : sb;
+ }
+
+ private static class ConstantStringWrapper {
+ private List<String> types;
+ private String prefix;
+ private String postfix;
+
+ ConstantStringWrapper(String type, String prefix, String postfix) {
+ this(Lists.newArrayList(type), prefix, postfix);
}
- sql.delete(sql.length() - 2, sql.length());
+ ConstantStringWrapper(List<String> types, String prefix, String postfix) {
+ this.types = types;
+ this.prefix = prefix;
+ this.postfix = postfix;
+ }
- sql.append(")");
+ public Object apply(final String typeName, Object value) {
+ return Iterables.any(types, new Predicate<String>() {
+
+ @Override
+ public boolean apply(@Nullable String type) {
+ return typeName.startsWith(type);
+ }
+ }) ? prefix + value + postfix : value;
+ }
}
+ private static final String SINGLE_QUOTATION = "'";
+ private static List<ConstantStringWrapper> WRAPPERS = Lists.newArrayList(
+ new ConstantStringWrapper(Lists.newArrayList(
+ serdeConstants.STRING_TYPE_NAME, serdeConstants.CHAR_TYPE_NAME,
+ serdeConstants.VARCHAR_TYPE_NAME, serdeConstants.DATE_TYPE_NAME,
+ serdeConstants.TIMESTAMP_TYPE_NAME
+ ), SINGLE_QUOTATION, SINGLE_QUOTATION),
+ new ConstantStringWrapper(serdeConstants.DATE_TYPE_NAME, "to_date(", ")"),
+ new ConstantStringWrapper(serdeConstants.TIMESTAMP_TYPE_NAME, "to_timestamp(", ")")
+ );
+
+ private Object createConstantString(String typeName, Object value) {
+ for (ConstantStringWrapper wrapper : WRAPPERS) {
+ value = wrapper.apply(typeName, value);
+ }
+
+ return value;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/973f7119/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index 1313fdb..0dd1134 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -76,7 +76,9 @@ public class PhoenixStorageHandlerUtil {
DateFormat df = null;
for (int i = 0, limit = values.length; i < limit; i++) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
+ if (serdeConstants.STRING_TYPE_NAME.equals(typeName) ||
+ typeName.startsWith(serdeConstants.CHAR_TYPE_NAME) ||
+ typeName.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) {
results[i] = values[i];
} else if (serdeConstants.INT_TYPE_NAME.equals(typeName)) {
results[i] = new Integer(values[i]);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/973f7119/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
new file mode 100644
index 0000000..7f1a7c3
--- /dev/null
+++ b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
@@ -0,0 +1,87 @@
+package org.apache.phoenix.hive.query;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.junit.Assert.assertEquals;
+
+public class PhoenixQueryBuilderTest {
+ private IndexSearchCondition mockedIndexSearchCondition(String comparisionOp,
+ Object constantValue,
+ Object[] constantValues,
+ String columnName,
+ String typeString,
+ boolean isNot) {
+ IndexSearchCondition condition = mock(IndexSearchCondition.class);
+ when(condition.getComparisonOp()).thenReturn(comparisionOp);
+
+ ExprNodeConstantDesc constantDesc = mock(ExprNodeConstantDesc.class);
+ when(constantDesc.getValue()).thenReturn(constantValue);
+ when(condition.getConstantDesc()).thenReturn(constantDesc);
+
+ ExprNodeColumnDesc columnDesc = mock(ExprNodeColumnDesc.class);
+ when(columnDesc.getColumn()).thenReturn(columnName);
+ when(columnDesc.getTypeString()).thenReturn(typeString);
+ when(condition.getColumnDesc()).thenReturn(columnDesc);
+
+
+ if (ArrayUtils.isNotEmpty(constantValues)) {
+ ExprNodeConstantDesc[] constantDescs = new ExprNodeConstantDesc[constantValues.length];
+ for (int i = 0; i < constantDescs.length; i++) {
+ constantDescs[i] = mock(ExprNodeConstantDesc.class);
+ when(condition.getConstantDesc(i)).thenReturn(constantDescs[i]);
+ when(constantDescs[i].getValue()).thenReturn(constantValues[i]);
+ }
+ when(condition.getConstantDescs()).thenReturn(constantDescs);
+ }
+
+ when(condition.isNot()).thenReturn(isNot);
+
+ return condition;
+ }
+
+ @Test
+ public void testBuildQueryWithCharColumns() throws IOException {
+ final String tableName = "TEST_TABLE";
+ final String COLUMN_CHAR = "Column_Char";
+ final String COLUMN_VARCHAR = "Column_VChar";
+ final String expectedQueryPrefix = "select /*+ NO_CACHE */ " + COLUMN_CHAR + "," + COLUMN_VARCHAR +
+ " from TEST_TABLE where ";
+
+ JobConf jobConf = new JobConf();
+ List<String> readColumnList = Lists.newArrayList(COLUMN_CHAR, COLUMN_VARCHAR);
+ List<IndexSearchCondition> searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE", null, COLUMN_CHAR, "char(10)", false),
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE2", null, COLUMN_VARCHAR, "varchar(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char = 'CHAR_VALUE' and Column_VChar = 'CHAR_VALUE2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFIn", null,
+ new Object[]{"CHAR1", "CHAR2", "CHAR3"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char in ('CHAR1', 'CHAR2', 'CHAR3')",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFBetween", null,
+ new Object[]{"CHAR1", "CHAR2"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char between 'CHAR1' and 'CHAR2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+ }
+}
[09/18] phoenix git commit: PHOENIX-3408 arithmetic/mathematical
operations with Decimal columns failed in Hive with PheonixStorageHandler.
Posted by ss...@apache.org.
PHOENIX-3408 arithmetic/mathematical operations with Decimal columns failed in Hive with PheonixStorageHandler.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/bebcc552
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/bebcc552
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/bebcc552
Branch: refs/heads/4.x-HBase-0.98
Commit: bebcc552fb2bc40b8fda79dc82be7fc4c61945bc
Parents: 46d4bb4
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Wed Oct 26 19:26:06 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:58:36 2016 -0700
----------------------------------------------------------------------
.../PhoenixDecimalObjectInspector.java | 19 ++++++++++++++-----
.../PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebcc552/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 8afe10f..3853c18 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -21,6 +21,9 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import java.math.BigDecimal;
@@ -30,17 +33,25 @@ public class PhoenixDecimalObjectInspector extends
implements HiveDecimalObjectInspector {
public PhoenixDecimalObjectInspector() {
- super(TypeInfoFactory.decimalTypeInfo);
+ this(TypeInfoFactory.decimalTypeInfo);
+ }
+
+ public PhoenixDecimalObjectInspector(PrimitiveTypeInfo typeInfo) {
+ super(typeInfo);
}
@Override
public Object copyObject(Object o) {
- return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
+ return o == null ? null : new BigDecimal(o.toString());
}
@Override
public HiveDecimal getPrimitiveJavaObject(Object o) {
- return HiveDecimal.create((BigDecimal) o);
+ if (o == null) {
+ return null;
+ }
+
+ return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create((BigDecimal) o),(DecimalTypeInfo)typeInfo);
}
@Override
@@ -56,8 +67,6 @@ public class PhoenixDecimalObjectInspector extends
}
return value;
-
-// return super.getPrimitiveWritableObject(o);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/bebcc552/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 928dede..22be0fc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -111,7 +111,7 @@ public class PhoenixObjectInspectorFactory {
oi = new PhoenixTimestampObjectInspector();
break;
case DECIMAL:
- oi = new PhoenixDecimalObjectInspector();
+ oi = new PhoenixDecimalObjectInspector((PrimitiveTypeInfo) type);
break;
case BINARY:
oi = new PhoenixBinaryObjectInspector();
[11/18] phoenix git commit: PHOENIX-3422 PhoenixQueryBuilder doesn't
make value string correctly for char(/varchar) column type.
Posted by ss...@apache.org.
PHOENIX-3422 PhoenixQueryBuilder doesn't make value string correctly for char(/varchar) column type.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a225f5ff
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a225f5ff
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a225f5ff
Branch: refs/heads/4.x-HBase-0.98
Commit: a225f5ffe773dde7a7efc1ada1d6dbda9d667cdf
Parents: cf70820
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Fri Oct 28 17:13:23 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:58:46 2016 -0700
----------------------------------------------------------------------
phoenix-hive/pom.xml | 7 +-
.../phoenix/hive/query/PhoenixQueryBuilder.java | 129 ++++++++++---------
.../hive/util/PhoenixStorageHandlerUtil.java | 4 +-
.../hive/query/PhoenixQueryBuilderTest.java | 87 +++++++++++++
4 files changed, 163 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a225f5ff/phoenix-hive/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml
index 250db49..c36e737 100644
--- a/phoenix-hive/pom.xml
+++ b/phoenix-hive/pom.xml
@@ -110,7 +110,12 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a225f5ff/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
index 8e3a972..a38814d 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
@@ -19,7 +19,9 @@ package org.apache.phoenix.hive.query;
import com.google.common.base.CharMatcher;
import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -31,12 +33,9 @@ import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.hive.util.PhoenixUtil;
+import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -662,17 +661,17 @@ public class PhoenixQueryBuilder {
comparisonOp);
if (comparisonOp.endsWith("UDFOPEqual")) { // column = 1
- appendCondition(sql, " = ", typeName, constantValues[0]);
+ sql.append(" = ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) { // column >= 1
- appendCondition(sql, " >= ", typeName, constantValues[0]);
+ sql.append(" >= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPGreaterThan")) { // column > 1
- appendCondition(sql, " > ", typeName, constantValues[0]);
+ sql.append(" > ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) { // column <= 1
- appendCondition(sql, " <= ", typeName, constantValues[0]);
+ sql.append(" <= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPLessThan")) { // column < 1
- appendCondition(sql, " < ", typeName, constantValues[0]);
+ sql.append(" < ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPNotEqual")) { // column != 1
- appendCondition(sql, " != ", typeName, constantValues[0]);
+ sql.append(" != ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("GenericUDFBetween")) {
appendBetweenCondition(jobConf, sql, condition.isNot(), typeName, constantValues);
} else if (comparisonOp.endsWith("GenericUDFIn")) {
@@ -687,44 +686,16 @@ public class PhoenixQueryBuilder {
return conditionColumnList;
}
- protected void appendCondition(StringBuilder sql, String comparisonOp, String typeName,
- String conditionValue) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(comparisonOp).append(conditionValue);
- }
- }
-
protected void appendBetweenCondition(JobConf jobConf, StringBuilder sql, boolean isNot,
String typeName, String[] conditionValues) throws
IOException {
- if (isNot) {
- sql.append(" not between ");
- } else {
- sql.append(" between ");
- }
-
try {
- Arrays.sort(PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName,
- conditionValues));
-
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValues[0]).append("'").append(" and ").append
- ("'").append(conditionValues[1]).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValues[0]).append("')").append(" and ")
- .append("to_date('").append(conditionValues[1]).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValues[0]).append("')").append(" and" +
- " ").append("to_timestamp('").append(conditionValues[1]).append("')");
- } else {
- sql.append(conditionValues[0]).append(" and ").append(conditionValues[1]);
- }
+ Object[] typedValues = PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName, conditionValues);
+ Arrays.sort(typedValues);
+
+ appendIfNot(isNot, sql).append(" between ")
+ .append(Joiner.on(" and ").join(createConstantString(typeName, typedValues[0]),
+ createConstantString(typeName, typedValues[1])));
} catch (Exception e) {
throw new IOException(e);
}
@@ -732,29 +703,63 @@ public class PhoenixQueryBuilder {
protected void appendInCondition(StringBuilder sql, boolean isNot, String typeName, String[]
conditionValues) {
- if (isNot) {
- sql.append(" not in (");
- } else {
- sql.append(" in (");
+ List<Object> wrappedConstants = Lists.newArrayListWithCapacity(conditionValues.length);
+ for (String conditionValue : conditionValues) {
+ wrappedConstants.add(createConstantString(typeName, conditionValue));
}
- for (String conditionValue : conditionValues) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(conditionValue);
- }
+ appendIfNot(isNot, sql)
+ .append(" in (")
+ .append(Joiner.on(", ").join(wrappedConstants))
+ .append(")");
+ }
- sql.append(", ");
+ private StringBuilder appendIfNot(boolean isNot, StringBuilder sb) {
+ return isNot ? sb.append(" not") : sb;
+ }
+
+ private static class ConstantStringWrapper {
+ private List<String> types;
+ private String prefix;
+ private String postfix;
+
+ ConstantStringWrapper(String type, String prefix, String postfix) {
+ this(Lists.newArrayList(type), prefix, postfix);
}
- sql.delete(sql.length() - 2, sql.length());
+ ConstantStringWrapper(List<String> types, String prefix, String postfix) {
+ this.types = types;
+ this.prefix = prefix;
+ this.postfix = postfix;
+ }
- sql.append(")");
+ public Object apply(final String typeName, Object value) {
+ return Iterables.any(types, new Predicate<String>() {
+
+ @Override
+ public boolean apply(@Nullable String type) {
+ return typeName.startsWith(type);
+ }
+ }) ? prefix + value + postfix : value;
+ }
}
+ private static final String SINGLE_QUOTATION = "'";
+ private static List<ConstantStringWrapper> WRAPPERS = Lists.newArrayList(
+ new ConstantStringWrapper(Lists.newArrayList(
+ serdeConstants.STRING_TYPE_NAME, serdeConstants.CHAR_TYPE_NAME,
+ serdeConstants.VARCHAR_TYPE_NAME, serdeConstants.DATE_TYPE_NAME,
+ serdeConstants.TIMESTAMP_TYPE_NAME
+ ), SINGLE_QUOTATION, SINGLE_QUOTATION),
+ new ConstantStringWrapper(serdeConstants.DATE_TYPE_NAME, "to_date(", ")"),
+ new ConstantStringWrapper(serdeConstants.TIMESTAMP_TYPE_NAME, "to_timestamp(", ")")
+ );
+
+ private Object createConstantString(String typeName, Object value) {
+ for (ConstantStringWrapper wrapper : WRAPPERS) {
+ value = wrapper.apply(typeName, value);
+ }
+
+ return value;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a225f5ff/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index 1313fdb..0dd1134 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -76,7 +76,9 @@ public class PhoenixStorageHandlerUtil {
DateFormat df = null;
for (int i = 0, limit = values.length; i < limit; i++) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
+ if (serdeConstants.STRING_TYPE_NAME.equals(typeName) ||
+ typeName.startsWith(serdeConstants.CHAR_TYPE_NAME) ||
+ typeName.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) {
results[i] = values[i];
} else if (serdeConstants.INT_TYPE_NAME.equals(typeName)) {
results[i] = new Integer(values[i]);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a225f5ff/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
new file mode 100644
index 0000000..7f1a7c3
--- /dev/null
+++ b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
@@ -0,0 +1,87 @@
+package org.apache.phoenix.hive.query;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.junit.Assert.assertEquals;
+
+public class PhoenixQueryBuilderTest {
+ private IndexSearchCondition mockedIndexSearchCondition(String comparisionOp,
+ Object constantValue,
+ Object[] constantValues,
+ String columnName,
+ String typeString,
+ boolean isNot) {
+ IndexSearchCondition condition = mock(IndexSearchCondition.class);
+ when(condition.getComparisonOp()).thenReturn(comparisionOp);
+
+ ExprNodeConstantDesc constantDesc = mock(ExprNodeConstantDesc.class);
+ when(constantDesc.getValue()).thenReturn(constantValue);
+ when(condition.getConstantDesc()).thenReturn(constantDesc);
+
+ ExprNodeColumnDesc columnDesc = mock(ExprNodeColumnDesc.class);
+ when(columnDesc.getColumn()).thenReturn(columnName);
+ when(columnDesc.getTypeString()).thenReturn(typeString);
+ when(condition.getColumnDesc()).thenReturn(columnDesc);
+
+
+ if (ArrayUtils.isNotEmpty(constantValues)) {
+ ExprNodeConstantDesc[] constantDescs = new ExprNodeConstantDesc[constantValues.length];
+ for (int i = 0; i < constantDescs.length; i++) {
+ constantDescs[i] = mock(ExprNodeConstantDesc.class);
+ when(condition.getConstantDesc(i)).thenReturn(constantDescs[i]);
+ when(constantDescs[i].getValue()).thenReturn(constantValues[i]);
+ }
+ when(condition.getConstantDescs()).thenReturn(constantDescs);
+ }
+
+ when(condition.isNot()).thenReturn(isNot);
+
+ return condition;
+ }
+
+ @Test
+ public void testBuildQueryWithCharColumns() throws IOException {
+ final String tableName = "TEST_TABLE";
+ final String COLUMN_CHAR = "Column_Char";
+ final String COLUMN_VARCHAR = "Column_VChar";
+ final String expectedQueryPrefix = "select /*+ NO_CACHE */ " + COLUMN_CHAR + "," + COLUMN_VARCHAR +
+ " from TEST_TABLE where ";
+
+ JobConf jobConf = new JobConf();
+ List<String> readColumnList = Lists.newArrayList(COLUMN_CHAR, COLUMN_VARCHAR);
+ List<IndexSearchCondition> searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE", null, COLUMN_CHAR, "char(10)", false),
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE2", null, COLUMN_VARCHAR, "varchar(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char = 'CHAR_VALUE' and Column_VChar = 'CHAR_VALUE2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFIn", null,
+ new Object[]{"CHAR1", "CHAR2", "CHAR3"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char in ('CHAR1', 'CHAR2', 'CHAR3')",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFBetween", null,
+ new Object[]{"CHAR1", "CHAR2"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char between 'CHAR1' and 'CHAR2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+ }
+}
[06/18] phoenix git commit: PHOENIX-3423 PhoenixObjectInspector
doesn't have information on length of the column.
Posted by ss...@apache.org.
PHOENIX-3423 PhoenixObjectInspector doesn't have information on length of the column.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/b477f370
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/b477f370
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/b477f370
Branch: refs/heads/master
Commit: b477f370c0519d5f5385927d3809fb4a5433a2ec
Parents: a37403c
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Mon Oct 31 12:36:00 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:38:44 2016 -0700
----------------------------------------------------------------------
.../hive/objectinspector/PhoenixCharObjectInspector.java | 7 ++++++-
.../hive/objectinspector/PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b477f370/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
index 8d6aa8c..17222a2 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.hive.objectinspector;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
/**
@@ -29,7 +30,11 @@ public class PhoenixCharObjectInspector extends AbstractPhoenixObjectInspector<H
implements HiveCharObjectInspector {
public PhoenixCharObjectInspector() {
- super(TypeInfoFactory.charTypeInfo);
+ this(TypeInfoFactory.charTypeInfo);
+ }
+
+ public PhoenixCharObjectInspector(PrimitiveTypeInfo type) {
+ super(type);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b477f370/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 22be0fc..3a19ea7 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -102,7 +102,7 @@ public class PhoenixObjectInspectorFactory {
serdeParams.getEscapeChar());
break;
case CHAR:
- oi = new PhoenixCharObjectInspector();
+ oi = new PhoenixCharObjectInspector((PrimitiveTypeInfo)type);
break;
case DATE:
oi = new PhoenixDateObjectInspector();
[02/18] phoenix git commit: PHOENIX-3416 Memory leak in
PhoenixStorageHandler
Posted by ss...@apache.org.
PHOENIX-3416 Memory leak in PhoenixStorageHandler
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e1afbcce
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e1afbcce
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e1afbcce
Branch: refs/heads/master
Commit: e1afbcce546b822e81f7fa0fc37e0de6fe5b8d0c
Parents: 86f5160
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Thu Oct 27 20:50:53 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:37:05 2016 -0700
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 14 +---
.../hive/mapreduce/PhoenixInputFormat.java | 37 +++++----
.../hive/ppd/PhoenixPredicateDecomposer.java | 15 +++-
.../ppd/PhoenixPredicateDecomposerManager.java | 83 --------------------
4 files changed, 33 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e1afbcce/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index e8b5b19..2bc8ace 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -40,8 +40,6 @@ import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.mapreduce.PhoenixInputFormat;
import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
-import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import java.util.List;
@@ -176,19 +174,9 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
ExprNodeDesc predicate) {
PhoenixSerDe phoenixSerDe = (PhoenixSerDe) deserializer;
- String tableName = phoenixSerDe.getTableProperties().getProperty
- (PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf, tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Decomposing predicate with predicateKey : " + predicateKey);
- }
-
List<String> columnNameList = phoenixSerDe.getSerdeParams().getColumnNames();
- PhoenixPredicateDecomposer predicateDecomposer = PhoenixPredicateDecomposerManager
- .createPredicateDecomposer(predicateKey, columnNameList);
- return predicateDecomposer.decomposePredicate(predicate);
+ return PhoenixPredicateDecomposer.create(columnNameList).decomposePredicate(predicate);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e1afbcce/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
index c5f6d18..fd6a631 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
@@ -32,15 +32,14 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.RegionSizeCalculator;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -48,7 +47,6 @@ import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.query.PhoenixQueryBuilder;
import org.apache.phoenix.hive.util.PhoenixConnectionUtil;
@@ -62,6 +60,7 @@ import org.apache.phoenix.util.PhoenixRuntime;
import java.io.IOException;
import java.sql.Connection;
import java.sql.Statement;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -83,8 +82,8 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
@Override
public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
- String tableName = jobConf.get(PhoenixConfigurationUtil.INPUT_TABLE_NAME);
- List<IndexSearchCondition> conditionList = null;
+ String tableName = jobConf.get(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
+
String query;
String executionEngine = jobConf.get(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname,
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.getDefaultValue());
@@ -97,17 +96,17 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
}
if (PhoenixStorageHandlerConstants.MR.equals(executionEngine)) {
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf,
- tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("PredicateKey for MR job : " + predicateKey);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer =
- PhoenixPredicateDecomposerManager.getPredicateDecomposer(predicateKey);
- if (predicateDecomposer != null && predicateDecomposer.isCalledPPD()) {
- conditionList = predicateDecomposer.getSearchConditionList();
+ List<IndexSearchCondition> conditionList = null;
+ String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
+ if (filterExprSerialized != null) {
+ ExprNodeGenericFuncDesc filterExpr =
+ Utilities.deserializeExpression(filterExprSerialized);
+ PhoenixPredicateDecomposer predicateDecomposer =
+ PhoenixPredicateDecomposer.create(Arrays.asList(jobConf.get(serdeConstants.LIST_COLUMNS).split(",")));
+ predicateDecomposer.decomposePredicate(filterExpr);
+ if (predicateDecomposer.isCalledPPD()) {
+ conditionList = predicateDecomposer.getSearchConditionList();
+ }
}
query = PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e1afbcce/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
index b94e4df..1e65819 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
@@ -42,11 +42,19 @@ public class PhoenixPredicateDecomposer {
private List<IndexSearchCondition> searchConditionList;
- public PhoenixPredicateDecomposer(List<String> columnNameList) {
+ public static PhoenixPredicateDecomposer create(List<String> columnNameList) {
+ return new PhoenixPredicateDecomposer(columnNameList);
+ }
+
+ private PhoenixPredicateDecomposer(List<String> columnNameList) {
this.columnNameList = columnNameList;
}
public DecomposedPredicate decomposePredicate(ExprNodeDesc predicate) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("predicate - " + predicate.toString());
+ }
+
IndexPredicateAnalyzer analyzer = PredicateAnalyzerFactory.createPredicateAnalyzer
(columnNameList, getFieldValidator());
DecomposedPredicate decomposed = new DecomposedPredicate();
@@ -65,6 +73,11 @@ public class PhoenixPredicateDecomposer {
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("decomposed predicate - residualPredicate: " + decomposed.residualPredicate +
+ ", pushedPredicate: " + decomposed.pushedPredicate);
+ }
+
return decomposed;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e1afbcce/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
deleted file mode 100644
index 2faef73..0000000
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.hive.ppd;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Support class that produces PredicateDecomposer for PhoenixStorageHandler
- */
-
-public class PhoenixPredicateDecomposerManager {
-
- private static final Log LOG = LogFactory.getLog(PhoenixPredicateDecomposerManager.class);
-
- // In case of absence of WHERE clause, PhoenixPredicateDecomposer is not created because
- // it's not called method of StorageHandler.decomposePredicate.
-
- private static final Map<String, List<PhoenixPredicateDecomposer>> PREDICATE_DECOMPOSER_MAP =
- Maps.newConcurrentMap();
-
- public static PhoenixPredicateDecomposer createPredicateDecomposer(String predicateKey,
- List<String>
- columnNameList) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
- if (predicateDecomposerList == null) {
- predicateDecomposerList = Lists.newArrayList();
- PREDICATE_DECOMPOSER_MAP.put(predicateKey, predicateDecomposerList);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer = new PhoenixPredicateDecomposer
- (columnNameList);
- predicateDecomposerList.add(predicateDecomposer);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" +
- predicateKey + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- public static PhoenixPredicateDecomposer getPredicateDecomposer(String predicateKey) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
-
- PhoenixPredicateDecomposer predicateDecomposer = null;
- if (predicateDecomposerList != null && predicateDecomposerList.size() > 0) {
- predicateDecomposer = predicateDecomposerList.remove(0);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" + predicateKey
- + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- private PhoenixPredicateDecomposerManager() {
- }
-}
[04/18] phoenix git commit: PHOENIX-3386 PhoenixStorageHandler throws
NPE if local tasks executed via child
Posted by ss...@apache.org.
PHOENIX-3386 PhoenixStorageHandler throws NPE if local tasks executed via child
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/41c16a02
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/41c16a02
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/41c16a02
Branch: refs/heads/master
Commit: 41c16a020fcd1cb143675ea17e2d9d3a56750a8a
Parents: c83d272
Author: Sergey Soldatov <ss...@apache.org>
Authored: Mon Oct 24 22:11:52 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:37:17 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java | 2 ++
.../java/org/apache/phoenix/hive/PhoenixStorageHandler.java | 4 ++++
.../org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java | 5 ++---
3 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/41c16a02/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index b1879d1..2264acd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -55,6 +55,8 @@ import com.google.common.collect.Lists;
public final class PhoenixConfigurationUtil {
private static final Log LOG = LogFactory.getLog(PhoenixInputFormat.class);
+
+ public static final String SESSION_ID = "phoenix.sessionid";
public static final String UPSERT_STATEMENT = "phoenix.upsert.stmt";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/41c16a02/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index 2bc8ace..bda2282 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.metadata.InputEstimator;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.mapred.InputFormat;
@@ -142,7 +143,10 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
tableProperties.setProperty(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME,
tableName);
}
+ SessionState sessionState = SessionState.get();
+ String sessionId = sessionState.getSessionId();
+ jobProperties.put(PhoenixConfigurationUtil.SESSION_ID, sessionId);
jobProperties.put(PhoenixConfigurationUtil.INPUT_TABLE_NAME, tableName);
jobProperties.put(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM, tableProperties
.getProperty(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/41c16a02/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index eb5fd24..1313fdb 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.net.DNS;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import javax.naming.NamingException;
import java.io.ByteArrayInputStream;
@@ -182,10 +183,8 @@ public class PhoenixStorageHandlerUtil {
}
public static String getTableKeyOfSession(JobConf jobConf, String tableName) {
- SessionState sessionState = SessionState.get();
-
- String sessionId = sessionState.getSessionId();
+ String sessionId = jobConf.get(PhoenixConfigurationUtil.SESSION_ID);
return new StringBuilder("[").append(sessionId).append("]-").append(tableName).toString();
}
[03/18] phoenix git commit: PHOENIX-3408 arithmetic/mathematical
operations with Decimal columns failed in Hive with PheonixStorageHandler.
Posted by ss...@apache.org.
PHOENIX-3408 arithmetic/mathematical operations with Decimal columns failed in Hive with PheonixStorageHandler.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c83d272b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c83d272b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c83d272b
Branch: refs/heads/master
Commit: c83d272b565447d39c42a4a8d3b0687bb2b5a16c
Parents: e1afbcc
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Wed Oct 26 19:26:06 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:37:12 2016 -0700
----------------------------------------------------------------------
.../PhoenixDecimalObjectInspector.java | 19 ++++++++++++++-----
.../PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c83d272b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 8afe10f..3853c18 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -21,6 +21,9 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import java.math.BigDecimal;
@@ -30,17 +33,25 @@ public class PhoenixDecimalObjectInspector extends
implements HiveDecimalObjectInspector {
public PhoenixDecimalObjectInspector() {
- super(TypeInfoFactory.decimalTypeInfo);
+ this(TypeInfoFactory.decimalTypeInfo);
+ }
+
+ public PhoenixDecimalObjectInspector(PrimitiveTypeInfo typeInfo) {
+ super(typeInfo);
}
@Override
public Object copyObject(Object o) {
- return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
+ return o == null ? null : new BigDecimal(o.toString());
}
@Override
public HiveDecimal getPrimitiveJavaObject(Object o) {
- return HiveDecimal.create((BigDecimal) o);
+ if (o == null) {
+ return null;
+ }
+
+ return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create((BigDecimal) o),(DecimalTypeInfo)typeInfo);
}
@Override
@@ -56,8 +67,6 @@ public class PhoenixDecimalObjectInspector extends
}
return value;
-
-// return super.getPrimitiveWritableObject(o);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c83d272b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 928dede..22be0fc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -111,7 +111,7 @@ public class PhoenixObjectInspectorFactory {
oi = new PhoenixTimestampObjectInspector();
break;
case DECIMAL:
- oi = new PhoenixDecimalObjectInspector();
+ oi = new PhoenixDecimalObjectInspector((PrimitiveTypeInfo) type);
break;
case BINARY:
oi = new PhoenixBinaryObjectInspector();
[07/18] phoenix git commit: PHOENIX-3387 Hive PhoenixStorageHandler
fails with join on numeric fields
Posted by ss...@apache.org.
PHOENIX-3387 Hive PhoenixStorageHandler fails with join on numeric fields
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3bb1a2b1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3bb1a2b1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3bb1a2b1
Branch: refs/heads/4.x-HBase-0.98
Commit: 3bb1a2b15ceee9d9b6c2f0e5fd66b8dcfb919d70
Parents: 1ed90b6
Author: Sergey Soldatov <ss...@apache.org>
Authored: Thu Oct 20 23:42:39 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:58:21 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/hive/HiveTestUtil.java | 1 +
.../objectinspector/PhoenixBooleanObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixByteObjectInspector.java | 5 +++++
.../objectinspector/PhoenixDecimalObjectInspector.java | 4 ++--
.../objectinspector/PhoenixDoubleObjectInspector.java | 5 +++++
.../objectinspector/PhoenixFloatObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixIntObjectInspector.java | 11 +++++++++++
.../hive/objectinspector/PhoenixLongObjectInspector.java | 5 +++++
.../objectinspector/PhoenixShortObjectInspector.java | 7 ++++++-
9 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
index a234d24..3407ffb 100644
--- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
+++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
@@ -567,6 +567,7 @@ public class HiveTestUtil {
public void init() throws Exception {
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ conf.setBoolVar(HiveConf.ConfVars.SUBMITLOCALTASKVIACHILD, false);
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
SessionState.start(conf);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
index 0795e14..a767ca0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixBooleanObjectInspector extends AbstractPhoenixObjectInspecto
}
@Override
+ public BooleanWritable getPrimitiveWritableObject(Object o) {
+ return new BooleanWritable(get(o));
+ }
+
+ @Override
public boolean get(Object o) {
Boolean value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
index c6c5e95..a19342a 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixByteObjectInspector extends AbstractPhoenixObjectInspector<B
}
@Override
+ public ByteWritable getPrimitiveWritableObject(Object o) {
+ return new ByteWritable(get(o));
+ }
+
+ @Override
public byte get(Object o) {
Byte value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 388863a..8afe10f 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -35,7 +35,7 @@ public class PhoenixDecimalObjectInspector extends
@Override
public Object copyObject(Object o) {
- return o == null ? null : new Decimal((Decimal) o);
+ return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
}
@Override
@@ -49,7 +49,7 @@ public class PhoenixDecimalObjectInspector extends
if (o != null) {
try {
- value = new HiveDecimalWritable((HiveDecimalWritable) o);
+ value = new HiveDecimalWritable(getPrimitiveJavaObject(o));
} catch (Exception e) {
logExceptionMessage(o, "DECIMAL");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
index 25ae793..9f440ed 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixDoubleObjectInspector extends AbstractPhoenixObjectInspector
}
@Override
+ public DoubleWritable getPrimitiveWritableObject(Object o) {
+ return new DoubleWritable(get(o));
+ }
+
+ @Override
public double get(Object o) {
Double value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
index 83ad2b0..bf1badc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
@@ -38,6 +38,11 @@ public class PhoenixFloatObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public FloatWritable getPrimitiveWritableObject(Object o) {
+ return new FloatWritable(get(o));
+ }
+
+ @Override
public float get(Object o) {
Float value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
index fc9e7d0..3511ee3 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.IntWritable;
@@ -34,6 +35,16 @@ public class PhoenixIntObjectInspector extends AbstractPhoenixObjectInspector<In
}
@Override
+ public Category getCategory() {
+ return Category.PRIMITIVE;
+ }
+
+ @Override
+ public IntWritable getPrimitiveWritableObject(Object o) {
+ return new IntWritable(get(o));
+ }
+
+ @Override
public int get(Object o) {
Integer value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
index ad5cd05..554f2a4 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixLongObjectInspector extends AbstractPhoenixObjectInspector<L
}
@Override
+ public LongWritable getPrimitiveWritableObject(Object o) {
+ return new LongWritable(get(o));
+ }
+
+ @Override
public long get(Object o) {
Long value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3bb1a2b1/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
index 1b7ec13..84529b0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
@@ -17,9 +17,9 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.ShortWritable;
public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<ShortWritable>
implements ShortObjectInspector {
@@ -34,6 +34,11 @@ public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public ShortWritable getPrimitiveWritableObject(Object o) {
+ return new ShortWritable(get(o));
+ }
+
+ @Override
public short get(Object o) {
Short value = null;
[12/18] phoenix git commit: PHOENIX-3423 PhoenixObjectInspector
doesn't have information on length of the column.
Posted by ss...@apache.org.
PHOENIX-3423 PhoenixObjectInspector doesn't have information on length of the column.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c1c78b2e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c1c78b2e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c1c78b2e
Branch: refs/heads/4.x-HBase-0.98
Commit: c1c78b2e41ced31017f978aa3fe356aaf7d42b7d
Parents: a225f5f
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Mon Oct 31 12:36:00 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:59:10 2016 -0700
----------------------------------------------------------------------
.../hive/objectinspector/PhoenixCharObjectInspector.java | 7 ++++++-
.../hive/objectinspector/PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1c78b2e/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
index 8d6aa8c..17222a2 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.hive.objectinspector;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
/**
@@ -29,7 +30,11 @@ public class PhoenixCharObjectInspector extends AbstractPhoenixObjectInspector<H
implements HiveCharObjectInspector {
public PhoenixCharObjectInspector() {
- super(TypeInfoFactory.charTypeInfo);
+ this(TypeInfoFactory.charTypeInfo);
+ }
+
+ public PhoenixCharObjectInspector(PrimitiveTypeInfo type) {
+ super(type);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c1c78b2e/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 22be0fc..3a19ea7 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -102,7 +102,7 @@ public class PhoenixObjectInspectorFactory {
serdeParams.getEscapeChar());
break;
case CHAR:
- oi = new PhoenixCharObjectInspector();
+ oi = new PhoenixCharObjectInspector((PrimitiveTypeInfo)type);
break;
case DATE:
oi = new PhoenixDateObjectInspector();
[18/18] phoenix git commit: PHOENIX-3423 PhoenixObjectInspector
doesn't have information on length of the column.
Posted by ss...@apache.org.
PHOENIX-3423 PhoenixObjectInspector doesn't have information on length of the column.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9aa18492
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9aa18492
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9aa18492
Branch: refs/heads/4.x-HBase-1.1
Commit: 9aa18492b54a54fdc9c03dc091bce52f68a9b216
Parents: 973f711
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Mon Oct 31 12:36:00 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:47 2016 -0700
----------------------------------------------------------------------
.../hive/objectinspector/PhoenixCharObjectInspector.java | 7 ++++++-
.../hive/objectinspector/PhoenixObjectInspectorFactory.java | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9aa18492/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
index 8d6aa8c..17222a2 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java
@@ -20,6 +20,7 @@ package org.apache.phoenix.hive.objectinspector;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
/**
@@ -29,7 +30,11 @@ public class PhoenixCharObjectInspector extends AbstractPhoenixObjectInspector<H
implements HiveCharObjectInspector {
public PhoenixCharObjectInspector() {
- super(TypeInfoFactory.charTypeInfo);
+ this(TypeInfoFactory.charTypeInfo);
+ }
+
+ public PhoenixCharObjectInspector(PrimitiveTypeInfo type) {
+ super(type);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9aa18492/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
index 22be0fc..3a19ea7 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixObjectInspectorFactory.java
@@ -102,7 +102,7 @@ public class PhoenixObjectInspectorFactory {
serdeParams.getEscapeChar());
break;
case CHAR:
- oi = new PhoenixCharObjectInspector();
+ oi = new PhoenixCharObjectInspector((PrimitiveTypeInfo)type);
break;
case DATE:
oi = new PhoenixDateObjectInspector();
[05/18] phoenix git commit: PHOENIX-3422 PhoenixQueryBuilder doesn't
make value string correctly for char(/varchar) column type.
Posted by ss...@apache.org.
PHOENIX-3422 PhoenixQueryBuilder doesn't make value string correctly for char(/varchar) column type.
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a37403cd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a37403cd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a37403cd
Branch: refs/heads/master
Commit: a37403cd9ff5c9c6913394fcbb18c97a6a267675
Parents: 41c16a0
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Fri Oct 28 17:13:23 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:37:24 2016 -0700
----------------------------------------------------------------------
phoenix-hive/pom.xml | 7 +-
.../phoenix/hive/query/PhoenixQueryBuilder.java | 129 ++++++++++---------
.../hive/util/PhoenixStorageHandlerUtil.java | 4 +-
.../hive/query/PhoenixQueryBuilderTest.java | 87 +++++++++++++
4 files changed, 163 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37403cd/phoenix-hive/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml
index 143bc09..9d4d2ab 100644
--- a/phoenix-hive/pom.xml
+++ b/phoenix-hive/pom.xml
@@ -110,7 +110,12 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito-all.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37403cd/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
index 8e3a972..a38814d 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/query/PhoenixQueryBuilder.java
@@ -19,7 +19,9 @@ package org.apache.phoenix.hive.query;
import com.google.common.base.CharMatcher;
import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -31,12 +33,9 @@ import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.hive.util.PhoenixUtil;
+import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -662,17 +661,17 @@ public class PhoenixQueryBuilder {
comparisonOp);
if (comparisonOp.endsWith("UDFOPEqual")) { // column = 1
- appendCondition(sql, " = ", typeName, constantValues[0]);
+ sql.append(" = ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) { // column >= 1
- appendCondition(sql, " >= ", typeName, constantValues[0]);
+ sql.append(" >= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPGreaterThan")) { // column > 1
- appendCondition(sql, " > ", typeName, constantValues[0]);
+ sql.append(" > ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) { // column <= 1
- appendCondition(sql, " <= ", typeName, constantValues[0]);
+ sql.append(" <= ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPLessThan")) { // column < 1
- appendCondition(sql, " < ", typeName, constantValues[0]);
+ sql.append(" < ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("UDFOPNotEqual")) { // column != 1
- appendCondition(sql, " != ", typeName, constantValues[0]);
+ sql.append(" != ").append(createConstantString(typeName, constantValues[0]));
} else if (comparisonOp.endsWith("GenericUDFBetween")) {
appendBetweenCondition(jobConf, sql, condition.isNot(), typeName, constantValues);
} else if (comparisonOp.endsWith("GenericUDFIn")) {
@@ -687,44 +686,16 @@ public class PhoenixQueryBuilder {
return conditionColumnList;
}
- protected void appendCondition(StringBuilder sql, String comparisonOp, String typeName,
- String conditionValue) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append(comparisonOp).append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(comparisonOp).append(conditionValue);
- }
- }
-
protected void appendBetweenCondition(JobConf jobConf, StringBuilder sql, boolean isNot,
String typeName, String[] conditionValues) throws
IOException {
- if (isNot) {
- sql.append(" not between ");
- } else {
- sql.append(" between ");
- }
-
try {
- Arrays.sort(PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName,
- conditionValues));
-
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValues[0]).append("'").append(" and ").append
- ("'").append(conditionValues[1]).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValues[0]).append("')").append(" and ")
- .append("to_date('").append(conditionValues[1]).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValues[0]).append("')").append(" and" +
- " ").append("to_timestamp('").append(conditionValues[1]).append("')");
- } else {
- sql.append(conditionValues[0]).append(" and ").append(conditionValues[1]);
- }
+ Object[] typedValues = PhoenixStorageHandlerUtil.toTypedValues(jobConf, typeName, conditionValues);
+ Arrays.sort(typedValues);
+
+ appendIfNot(isNot, sql).append(" between ")
+ .append(Joiner.on(" and ").join(createConstantString(typeName, typedValues[0]),
+ createConstantString(typeName, typedValues[1])));
} catch (Exception e) {
throw new IOException(e);
}
@@ -732,29 +703,63 @@ public class PhoenixQueryBuilder {
protected void appendInCondition(StringBuilder sql, boolean isNot, String typeName, String[]
conditionValues) {
- if (isNot) {
- sql.append(" not in (");
- } else {
- sql.append(" in (");
+ List<Object> wrappedConstants = Lists.newArrayListWithCapacity(conditionValues.length);
+ for (String conditionValue : conditionValues) {
+ wrappedConstants.add(createConstantString(typeName, conditionValue));
}
- for (String conditionValue : conditionValues) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
- sql.append("'").append(conditionValue).append("'");
- } else if (serdeConstants.DATE_TYPE_NAME.equals(typeName)) {
- sql.append("to_date('").append(conditionValue).append("')");
- } else if (serdeConstants.TIMESTAMP_TYPE_NAME.equals(typeName)) {
- sql.append("to_timestamp('").append(conditionValue).append("')");
- } else {
- sql.append(conditionValue);
- }
+ appendIfNot(isNot, sql)
+ .append(" in (")
+ .append(Joiner.on(", ").join(wrappedConstants))
+ .append(")");
+ }
- sql.append(", ");
+ private StringBuilder appendIfNot(boolean isNot, StringBuilder sb) {
+ return isNot ? sb.append(" not") : sb;
+ }
+
+ private static class ConstantStringWrapper {
+ private List<String> types;
+ private String prefix;
+ private String postfix;
+
+ ConstantStringWrapper(String type, String prefix, String postfix) {
+ this(Lists.newArrayList(type), prefix, postfix);
}
- sql.delete(sql.length() - 2, sql.length());
+ ConstantStringWrapper(List<String> types, String prefix, String postfix) {
+ this.types = types;
+ this.prefix = prefix;
+ this.postfix = postfix;
+ }
- sql.append(")");
+ public Object apply(final String typeName, Object value) {
+ return Iterables.any(types, new Predicate<String>() {
+
+ @Override
+ public boolean apply(@Nullable String type) {
+ return typeName.startsWith(type);
+ }
+ }) ? prefix + value + postfix : value;
+ }
}
+ private static final String SINGLE_QUOTATION = "'";
+ private static List<ConstantStringWrapper> WRAPPERS = Lists.newArrayList(
+ new ConstantStringWrapper(Lists.newArrayList(
+ serdeConstants.STRING_TYPE_NAME, serdeConstants.CHAR_TYPE_NAME,
+ serdeConstants.VARCHAR_TYPE_NAME, serdeConstants.DATE_TYPE_NAME,
+ serdeConstants.TIMESTAMP_TYPE_NAME
+ ), SINGLE_QUOTATION, SINGLE_QUOTATION),
+ new ConstantStringWrapper(serdeConstants.DATE_TYPE_NAME, "to_date(", ")"),
+ new ConstantStringWrapper(serdeConstants.TIMESTAMP_TYPE_NAME, "to_timestamp(", ")")
+ );
+
+ private Object createConstantString(String typeName, Object value) {
+ for (ConstantStringWrapper wrapper : WRAPPERS) {
+ value = wrapper.apply(typeName, value);
+ }
+
+ return value;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37403cd/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index 1313fdb..0dd1134 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -76,7 +76,9 @@ public class PhoenixStorageHandlerUtil {
DateFormat df = null;
for (int i = 0, limit = values.length; i < limit; i++) {
- if (serdeConstants.STRING_TYPE_NAME.equals(typeName)) {
+ if (serdeConstants.STRING_TYPE_NAME.equals(typeName) ||
+ typeName.startsWith(serdeConstants.CHAR_TYPE_NAME) ||
+ typeName.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) {
results[i] = values[i];
} else if (serdeConstants.INT_TYPE_NAME.equals(typeName)) {
results[i] = new Integer(values[i]);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a37403cd/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
new file mode 100644
index 0000000..7f1a7c3
--- /dev/null
+++ b/phoenix-hive/src/test/java/org/apache/phoenix/hive/query/PhoenixQueryBuilderTest.java
@@ -0,0 +1,87 @@
+package org.apache.phoenix.hive.query;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.junit.Assert.assertEquals;
+
+public class PhoenixQueryBuilderTest {
+ private IndexSearchCondition mockedIndexSearchCondition(String comparisionOp,
+ Object constantValue,
+ Object[] constantValues,
+ String columnName,
+ String typeString,
+ boolean isNot) {
+ IndexSearchCondition condition = mock(IndexSearchCondition.class);
+ when(condition.getComparisonOp()).thenReturn(comparisionOp);
+
+ ExprNodeConstantDesc constantDesc = mock(ExprNodeConstantDesc.class);
+ when(constantDesc.getValue()).thenReturn(constantValue);
+ when(condition.getConstantDesc()).thenReturn(constantDesc);
+
+ ExprNodeColumnDesc columnDesc = mock(ExprNodeColumnDesc.class);
+ when(columnDesc.getColumn()).thenReturn(columnName);
+ when(columnDesc.getTypeString()).thenReturn(typeString);
+ when(condition.getColumnDesc()).thenReturn(columnDesc);
+
+
+ if (ArrayUtils.isNotEmpty(constantValues)) {
+ ExprNodeConstantDesc[] constantDescs = new ExprNodeConstantDesc[constantValues.length];
+ for (int i = 0; i < constantDescs.length; i++) {
+ constantDescs[i] = mock(ExprNodeConstantDesc.class);
+ when(condition.getConstantDesc(i)).thenReturn(constantDescs[i]);
+ when(constantDescs[i].getValue()).thenReturn(constantValues[i]);
+ }
+ when(condition.getConstantDescs()).thenReturn(constantDescs);
+ }
+
+ when(condition.isNot()).thenReturn(isNot);
+
+ return condition;
+ }
+
+ @Test
+ public void testBuildQueryWithCharColumns() throws IOException {
+ final String tableName = "TEST_TABLE";
+ final String COLUMN_CHAR = "Column_Char";
+ final String COLUMN_VARCHAR = "Column_VChar";
+ final String expectedQueryPrefix = "select /*+ NO_CACHE */ " + COLUMN_CHAR + "," + COLUMN_VARCHAR +
+ " from TEST_TABLE where ";
+
+ JobConf jobConf = new JobConf();
+ List<String> readColumnList = Lists.newArrayList(COLUMN_CHAR, COLUMN_VARCHAR);
+ List<IndexSearchCondition> searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE", null, COLUMN_CHAR, "char(10)", false),
+ mockedIndexSearchCondition("GenericUDFOPEqual", "CHAR_VALUE2", null, COLUMN_VARCHAR, "varchar(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char = 'CHAR_VALUE' and Column_VChar = 'CHAR_VALUE2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFIn", null,
+ new Object[]{"CHAR1", "CHAR2", "CHAR3"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char in ('CHAR1', 'CHAR2', 'CHAR3')",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+
+ searchConditions = Lists.newArrayList(
+ mockedIndexSearchCondition("GenericUDFBetween", null,
+ new Object[]{"CHAR1", "CHAR2"}, COLUMN_CHAR, "char(10)", false)
+ );
+
+ assertEquals(expectedQueryPrefix + "Column_Char between 'CHAR1' and 'CHAR2'",
+ PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName, readColumnList, searchConditions));
+ }
+}
[10/18] phoenix git commit: PHOENIX-3386 PhoenixStorageHandler throws
NPE if local tasks executed via child
Posted by ss...@apache.org.
PHOENIX-3386 PhoenixStorageHandler throws NPE if local tasks executed via child
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cf70820b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cf70820b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cf70820b
Branch: refs/heads/4.x-HBase-0.98
Commit: cf70820b9dee6968ac26c66c5c98079158a48ac1
Parents: bebcc55
Author: Sergey Soldatov <ss...@apache.org>
Authored: Mon Oct 24 22:11:52 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:58:40 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java | 2 ++
.../java/org/apache/phoenix/hive/PhoenixStorageHandler.java | 4 ++++
.../org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java | 5 ++---
3 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf70820b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index b1879d1..2264acd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -55,6 +55,8 @@ import com.google.common.collect.Lists;
public final class PhoenixConfigurationUtil {
private static final Log LOG = LogFactory.getLog(PhoenixInputFormat.class);
+
+ public static final String SESSION_ID = "phoenix.sessionid";
public static final String UPSERT_STATEMENT = "phoenix.upsert.stmt";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf70820b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index 2bc8ace..bda2282 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.metadata.InputEstimator;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.mapred.InputFormat;
@@ -142,7 +143,10 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
tableProperties.setProperty(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME,
tableName);
}
+ SessionState sessionState = SessionState.get();
+ String sessionId = sessionState.getSessionId();
+ jobProperties.put(PhoenixConfigurationUtil.SESSION_ID, sessionId);
jobProperties.put(PhoenixConfigurationUtil.INPUT_TABLE_NAME, tableName);
jobProperties.put(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM, tableProperties
.getProperty(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf70820b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index eb5fd24..1313fdb 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.net.DNS;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import javax.naming.NamingException;
import java.io.ByteArrayInputStream;
@@ -182,10 +183,8 @@ public class PhoenixStorageHandlerUtil {
}
public static String getTableKeyOfSession(JobConf jobConf, String tableName) {
- SessionState sessionState = SessionState.get();
-
- String sessionId = sessionState.getSessionId();
+ String sessionId = jobConf.get(PhoenixConfigurationUtil.SESSION_ID);
return new StringBuilder("[").append(sessionId).append("]-").append(tableName).toString();
}
[13/18] phoenix git commit: PHOENIX-3387 Hive PhoenixStorageHandler
fails with join on numeric fields
Posted by ss...@apache.org.
PHOENIX-3387 Hive PhoenixStorageHandler fails with join on numeric fields
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d9f730a3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d9f730a3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d9f730a3
Branch: refs/heads/4.x-HBase-1.1
Commit: d9f730a35e65aee24f33d530bb0843a9160bc148
Parents: 4f0dd6a
Author: Sergey Soldatov <ss...@apache.org>
Authored: Thu Oct 20 23:42:39 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:31 2016 -0700
----------------------------------------------------------------------
.../it/java/org/apache/phoenix/hive/HiveTestUtil.java | 1 +
.../objectinspector/PhoenixBooleanObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixByteObjectInspector.java | 5 +++++
.../objectinspector/PhoenixDecimalObjectInspector.java | 4 ++--
.../objectinspector/PhoenixDoubleObjectInspector.java | 5 +++++
.../objectinspector/PhoenixFloatObjectInspector.java | 5 +++++
.../hive/objectinspector/PhoenixIntObjectInspector.java | 11 +++++++++++
.../hive/objectinspector/PhoenixLongObjectInspector.java | 5 +++++
.../objectinspector/PhoenixShortObjectInspector.java | 7 ++++++-
9 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
index a234d24..3407ffb 100644
--- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
+++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java
@@ -567,6 +567,7 @@ public class HiveTestUtil {
public void init() throws Exception {
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ conf.setBoolVar(HiveConf.ConfVars.SUBMITLOCALTASKVIACHILD, false);
String execEngine = conf.get("hive.execution.engine");
conf.set("hive.execution.engine", "mr");
SessionState.start(conf);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
index 0795e14..a767ca0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixBooleanObjectInspector extends AbstractPhoenixObjectInspecto
}
@Override
+ public BooleanWritable getPrimitiveWritableObject(Object o) {
+ return new BooleanWritable(get(o));
+ }
+
+ @Override
public boolean get(Object o) {
Boolean value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
index c6c5e95..a19342a 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixByteObjectInspector extends AbstractPhoenixObjectInspector<B
}
@Override
+ public ByteWritable getPrimitiveWritableObject(Object o) {
+ return new ByteWritable(get(o));
+ }
+
+ @Override
public byte get(Object o) {
Byte value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
index 388863a..8afe10f 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDecimalObjectInspector.java
@@ -35,7 +35,7 @@ public class PhoenixDecimalObjectInspector extends
@Override
public Object copyObject(Object o) {
- return o == null ? null : new Decimal((Decimal) o);
+ return o == null ? null : new BigDecimal(((BigDecimal)o).byteValue());
}
@Override
@@ -49,7 +49,7 @@ public class PhoenixDecimalObjectInspector extends
if (o != null) {
try {
- value = new HiveDecimalWritable((HiveDecimalWritable) o);
+ value = new HiveDecimalWritable(getPrimitiveJavaObject(o));
} catch (Exception e) {
logExceptionMessage(o, "DECIMAL");
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
index 25ae793..9f440ed 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java
@@ -37,6 +37,11 @@ public class PhoenixDoubleObjectInspector extends AbstractPhoenixObjectInspector
}
@Override
+ public DoubleWritable getPrimitiveWritableObject(Object o) {
+ return new DoubleWritable(get(o));
+ }
+
+ @Override
public double get(Object o) {
Double value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
index 83ad2b0..bf1badc 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java
@@ -38,6 +38,11 @@ public class PhoenixFloatObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public FloatWritable getPrimitiveWritableObject(Object o) {
+ return new FloatWritable(get(o));
+ }
+
+ @Override
public float get(Object o) {
Float value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
index fc9e7d0..3511ee3 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.IntWritable;
@@ -34,6 +35,16 @@ public class PhoenixIntObjectInspector extends AbstractPhoenixObjectInspector<In
}
@Override
+ public Category getCategory() {
+ return Category.PRIMITIVE;
+ }
+
+ @Override
+ public IntWritable getPrimitiveWritableObject(Object o) {
+ return new IntWritable(get(o));
+ }
+
+ @Override
public int get(Object o) {
Integer value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
index ad5cd05..554f2a4 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java
@@ -34,6 +34,11 @@ public class PhoenixLongObjectInspector extends AbstractPhoenixObjectInspector<L
}
@Override
+ public LongWritable getPrimitiveWritableObject(Object o) {
+ return new LongWritable(get(o));
+ }
+
+ @Override
public long get(Object o) {
Long value = null;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d9f730a3/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
index 1b7ec13..84529b0 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java
@@ -17,9 +17,9 @@
*/
package org.apache.phoenix.hive.objectinspector;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.ShortWritable;
public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<ShortWritable>
implements ShortObjectInspector {
@@ -34,6 +34,11 @@ public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector<
}
@Override
+ public ShortWritable getPrimitiveWritableObject(Object o) {
+ return new ShortWritable(get(o));
+ }
+
+ @Override
public short get(Object o) {
Short value = null;
[16/18] phoenix git commit: PHOENIX-3386 PhoenixStorageHandler throws
NPE if local tasks executed via child
Posted by ss...@apache.org.
PHOENIX-3386 PhoenixStorageHandler throws NPE if local tasks executed via child
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/f138800e
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/f138800e
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/f138800e
Branch: refs/heads/4.x-HBase-1.1
Commit: f138800e779bbaf8c035942b699a8bea627ac8b7
Parents: d5f7393
Author: Sergey Soldatov <ss...@apache.org>
Authored: Mon Oct 24 22:11:52 2016 -0700
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:40 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java | 2 ++
.../java/org/apache/phoenix/hive/PhoenixStorageHandler.java | 4 ++++
.../org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java | 5 ++---
3 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f138800e/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index b1879d1..2264acd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -55,6 +55,8 @@ import com.google.common.collect.Lists;
public final class PhoenixConfigurationUtil {
private static final Log LOG = LogFactory.getLog(PhoenixInputFormat.class);
+
+ public static final String SESSION_ID = "phoenix.sessionid";
public static final String UPSERT_STATEMENT = "phoenix.upsert.stmt";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f138800e/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index 2bc8ace..bda2282 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveStoragePredicateHandler;
import org.apache.hadoop.hive.ql.metadata.InputEstimator;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.mapred.InputFormat;
@@ -142,7 +143,10 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
tableProperties.setProperty(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME,
tableName);
}
+ SessionState sessionState = SessionState.get();
+ String sessionId = sessionState.getSessionId();
+ jobProperties.put(PhoenixConfigurationUtil.SESSION_ID, sessionId);
jobProperties.put(PhoenixConfigurationUtil.INPUT_TABLE_NAME, tableName);
jobProperties.put(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM, tableProperties
.getProperty(PhoenixStorageHandlerConstants.ZOOKEEPER_QUORUM,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/f138800e/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
index eb5fd24..1313fdb 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/util/PhoenixStorageHandlerUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.net.DNS;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import javax.naming.NamingException;
import java.io.ByteArrayInputStream;
@@ -182,10 +183,8 @@ public class PhoenixStorageHandlerUtil {
}
public static String getTableKeyOfSession(JobConf jobConf, String tableName) {
- SessionState sessionState = SessionState.get();
-
- String sessionId = sessionState.getSessionId();
+ String sessionId = jobConf.get(PhoenixConfigurationUtil.SESSION_ID);
return new StringBuilder("[").append(sessionId).append("]-").append(tableName).toString();
}
[08/18] phoenix git commit: PHOENIX-3416 Memory leak in
PhoenixStorageHandler
Posted by ss...@apache.org.
PHOENIX-3416 Memory leak in PhoenixStorageHandler
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/46d4bb4c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/46d4bb4c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/46d4bb4c
Branch: refs/heads/4.x-HBase-0.98
Commit: 46d4bb4ca0a9f90316c3f36d397b36405d8766e7
Parents: 3bb1a2b
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Thu Oct 27 20:50:53 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 12:58:32 2016 -0700
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 14 +---
.../hive/mapreduce/PhoenixInputFormat.java | 37 +++++----
.../hive/ppd/PhoenixPredicateDecomposer.java | 15 +++-
.../ppd/PhoenixPredicateDecomposerManager.java | 83 --------------------
4 files changed, 33 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/46d4bb4c/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index e8b5b19..2bc8ace 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -40,8 +40,6 @@ import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.mapreduce.PhoenixInputFormat;
import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
-import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import java.util.List;
@@ -176,19 +174,9 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
ExprNodeDesc predicate) {
PhoenixSerDe phoenixSerDe = (PhoenixSerDe) deserializer;
- String tableName = phoenixSerDe.getTableProperties().getProperty
- (PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf, tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Decomposing predicate with predicateKey : " + predicateKey);
- }
-
List<String> columnNameList = phoenixSerDe.getSerdeParams().getColumnNames();
- PhoenixPredicateDecomposer predicateDecomposer = PhoenixPredicateDecomposerManager
- .createPredicateDecomposer(predicateKey, columnNameList);
- return predicateDecomposer.decomposePredicate(predicate);
+ return PhoenixPredicateDecomposer.create(columnNameList).decomposePredicate(predicate);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/46d4bb4c/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
index 0944bb7..e3d0212 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
@@ -32,15 +32,14 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.RegionSizeCalculator;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -48,7 +47,6 @@ import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.query.PhoenixQueryBuilder;
import org.apache.phoenix.hive.util.PhoenixConnectionUtil;
@@ -62,6 +60,7 @@ import org.apache.phoenix.util.PhoenixRuntime;
import java.io.IOException;
import java.sql.Connection;
import java.sql.Statement;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -83,8 +82,8 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
@Override
public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
- String tableName = jobConf.get(PhoenixConfigurationUtil.INPUT_TABLE_NAME);
- List<IndexSearchCondition> conditionList = null;
+ String tableName = jobConf.get(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
+
String query;
String executionEngine = jobConf.get(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname,
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.getDefaultValue());
@@ -97,17 +96,17 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
}
if (PhoenixStorageHandlerConstants.MR.equals(executionEngine)) {
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf,
- tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("PredicateKey for MR job : " + predicateKey);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer =
- PhoenixPredicateDecomposerManager.getPredicateDecomposer(predicateKey);
- if (predicateDecomposer != null && predicateDecomposer.isCalledPPD()) {
- conditionList = predicateDecomposer.getSearchConditionList();
+ List<IndexSearchCondition> conditionList = null;
+ String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
+ if (filterExprSerialized != null) {
+ ExprNodeGenericFuncDesc filterExpr =
+ Utilities.deserializeExpression(filterExprSerialized);
+ PhoenixPredicateDecomposer predicateDecomposer =
+ PhoenixPredicateDecomposer.create(Arrays.asList(jobConf.get(serdeConstants.LIST_COLUMNS).split(",")));
+ predicateDecomposer.decomposePredicate(filterExpr);
+ if (predicateDecomposer.isCalledPPD()) {
+ conditionList = predicateDecomposer.getSearchConditionList();
+ }
}
query = PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/46d4bb4c/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
index b94e4df..1e65819 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
@@ -42,11 +42,19 @@ public class PhoenixPredicateDecomposer {
private List<IndexSearchCondition> searchConditionList;
- public PhoenixPredicateDecomposer(List<String> columnNameList) {
+ public static PhoenixPredicateDecomposer create(List<String> columnNameList) {
+ return new PhoenixPredicateDecomposer(columnNameList);
+ }
+
+ private PhoenixPredicateDecomposer(List<String> columnNameList) {
this.columnNameList = columnNameList;
}
public DecomposedPredicate decomposePredicate(ExprNodeDesc predicate) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("predicate - " + predicate.toString());
+ }
+
IndexPredicateAnalyzer analyzer = PredicateAnalyzerFactory.createPredicateAnalyzer
(columnNameList, getFieldValidator());
DecomposedPredicate decomposed = new DecomposedPredicate();
@@ -65,6 +73,11 @@ public class PhoenixPredicateDecomposer {
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("decomposed predicate - residualPredicate: " + decomposed.residualPredicate +
+ ", pushedPredicate: " + decomposed.pushedPredicate);
+ }
+
return decomposed;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/46d4bb4c/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
deleted file mode 100644
index 2faef73..0000000
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.hive.ppd;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Support class that produces PredicateDecomposer for PhoenixStorageHandler
- */
-
-public class PhoenixPredicateDecomposerManager {
-
- private static final Log LOG = LogFactory.getLog(PhoenixPredicateDecomposerManager.class);
-
- // In case of absence of WHERE clause, PhoenixPredicateDecomposer is not created because
- // it's not called method of StorageHandler.decomposePredicate.
-
- private static final Map<String, List<PhoenixPredicateDecomposer>> PREDICATE_DECOMPOSER_MAP =
- Maps.newConcurrentMap();
-
- public static PhoenixPredicateDecomposer createPredicateDecomposer(String predicateKey,
- List<String>
- columnNameList) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
- if (predicateDecomposerList == null) {
- predicateDecomposerList = Lists.newArrayList();
- PREDICATE_DECOMPOSER_MAP.put(predicateKey, predicateDecomposerList);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer = new PhoenixPredicateDecomposer
- (columnNameList);
- predicateDecomposerList.add(predicateDecomposer);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" +
- predicateKey + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- public static PhoenixPredicateDecomposer getPredicateDecomposer(String predicateKey) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
-
- PhoenixPredicateDecomposer predicateDecomposer = null;
- if (predicateDecomposerList != null && predicateDecomposerList.size() > 0) {
- predicateDecomposer = predicateDecomposerList.remove(0);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" + predicateKey
- + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- private PhoenixPredicateDecomposerManager() {
- }
-}
[14/18] phoenix git commit: PHOENIX-3416 Memory leak in
PhoenixStorageHandler
Posted by ss...@apache.org.
PHOENIX-3416 Memory leak in PhoenixStorageHandler
Signed-off-by: Sergey Soldatov <ss...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8463d3d8
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8463d3d8
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8463d3d8
Branch: refs/heads/4.x-HBase-1.1
Commit: 8463d3d870e422e1c50175e6c28b6e51ccdb4488
Parents: d9f730a
Author: Jeongdae Kim <kj...@gmail.com>
Authored: Thu Oct 27 20:50:53 2016 +0900
Committer: Sergey Soldatov <ss...@apache.org>
Committed: Wed Nov 2 13:01:34 2016 -0700
----------------------------------------------------------------------
.../phoenix/hive/PhoenixStorageHandler.java | 14 +---
.../hive/mapreduce/PhoenixInputFormat.java | 37 +++++----
.../hive/ppd/PhoenixPredicateDecomposer.java | 15 +++-
.../ppd/PhoenixPredicateDecomposerManager.java | 83 --------------------
4 files changed, 33 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8463d3d8/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
index e8b5b19..2bc8ace 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/PhoenixStorageHandler.java
@@ -40,8 +40,6 @@ import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.mapreduce.PhoenixInputFormat;
import org.apache.phoenix.hive.mapreduce.PhoenixOutputFormat;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
-import org.apache.phoenix.hive.util.PhoenixStorageHandlerUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import java.util.List;
@@ -176,19 +174,9 @@ public class PhoenixStorageHandler extends DefaultStorageHandler implements
public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
ExprNodeDesc predicate) {
PhoenixSerDe phoenixSerDe = (PhoenixSerDe) deserializer;
- String tableName = phoenixSerDe.getTableProperties().getProperty
- (PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf, tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Decomposing predicate with predicateKey : " + predicateKey);
- }
-
List<String> columnNameList = phoenixSerDe.getSerdeParams().getColumnNames();
- PhoenixPredicateDecomposer predicateDecomposer = PhoenixPredicateDecomposerManager
- .createPredicateDecomposer(predicateKey, columnNameList);
- return predicateDecomposer.decomposePredicate(predicate);
+ return PhoenixPredicateDecomposer.create(columnNameList).decomposePredicate(predicate);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8463d3d8/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
index c5f6d18..fd6a631 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/mapreduce/PhoenixInputFormat.java
@@ -32,15 +32,14 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.RegionSizeCalculator;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.db.DBWritable;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
@@ -48,7 +47,6 @@ import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.hive.constants.PhoenixStorageHandlerConstants;
import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposer;
-import org.apache.phoenix.hive.ppd.PhoenixPredicateDecomposerManager;
import org.apache.phoenix.hive.ql.index.IndexSearchCondition;
import org.apache.phoenix.hive.query.PhoenixQueryBuilder;
import org.apache.phoenix.hive.util.PhoenixConnectionUtil;
@@ -62,6 +60,7 @@ import org.apache.phoenix.util.PhoenixRuntime;
import java.io.IOException;
import java.sql.Connection;
import java.sql.Statement;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -83,8 +82,8 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
@Override
public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
- String tableName = jobConf.get(PhoenixConfigurationUtil.INPUT_TABLE_NAME);
- List<IndexSearchCondition> conditionList = null;
+ String tableName = jobConf.get(PhoenixStorageHandlerConstants.PHOENIX_TABLE_NAME);
+
String query;
String executionEngine = jobConf.get(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname,
HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.getDefaultValue());
@@ -97,17 +96,17 @@ public class PhoenixInputFormat<T extends DBWritable> implements InputFormat<Wri
}
if (PhoenixStorageHandlerConstants.MR.equals(executionEngine)) {
- String predicateKey = PhoenixStorageHandlerUtil.getTableKeyOfSession(jobConf,
- tableName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("PredicateKey for MR job : " + predicateKey);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer =
- PhoenixPredicateDecomposerManager.getPredicateDecomposer(predicateKey);
- if (predicateDecomposer != null && predicateDecomposer.isCalledPPD()) {
- conditionList = predicateDecomposer.getSearchConditionList();
+ List<IndexSearchCondition> conditionList = null;
+ String filterExprSerialized = jobConf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
+ if (filterExprSerialized != null) {
+ ExprNodeGenericFuncDesc filterExpr =
+ Utilities.deserializeExpression(filterExprSerialized);
+ PhoenixPredicateDecomposer predicateDecomposer =
+ PhoenixPredicateDecomposer.create(Arrays.asList(jobConf.get(serdeConstants.LIST_COLUMNS).split(",")));
+ predicateDecomposer.decomposePredicate(filterExpr);
+ if (predicateDecomposer.isCalledPPD()) {
+ conditionList = predicateDecomposer.getSearchConditionList();
+ }
}
query = PhoenixQueryBuilder.getInstance().buildQuery(jobConf, tableName,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8463d3d8/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
index b94e4df..1e65819 100644
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
+++ b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposer.java
@@ -42,11 +42,19 @@ public class PhoenixPredicateDecomposer {
private List<IndexSearchCondition> searchConditionList;
- public PhoenixPredicateDecomposer(List<String> columnNameList) {
+ public static PhoenixPredicateDecomposer create(List<String> columnNameList) {
+ return new PhoenixPredicateDecomposer(columnNameList);
+ }
+
+ private PhoenixPredicateDecomposer(List<String> columnNameList) {
this.columnNameList = columnNameList;
}
public DecomposedPredicate decomposePredicate(ExprNodeDesc predicate) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("predicate - " + predicate.toString());
+ }
+
IndexPredicateAnalyzer analyzer = PredicateAnalyzerFactory.createPredicateAnalyzer
(columnNameList, getFieldValidator());
DecomposedPredicate decomposed = new DecomposedPredicate();
@@ -65,6 +73,11 @@ public class PhoenixPredicateDecomposer {
}
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("decomposed predicate - residualPredicate: " + decomposed.residualPredicate +
+ ", pushedPredicate: " + decomposed.pushedPredicate);
+ }
+
return decomposed;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/8463d3d8/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java b/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
deleted file mode 100644
index 2faef73..0000000
--- a/phoenix-hive/src/main/java/org/apache/phoenix/hive/ppd/PhoenixPredicateDecomposerManager.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.phoenix.hive.ppd;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Support class that produces PredicateDecomposer for PhoenixStorageHandler
- */
-
-public class PhoenixPredicateDecomposerManager {
-
- private static final Log LOG = LogFactory.getLog(PhoenixPredicateDecomposerManager.class);
-
- // In case of absence of WHERE clause, PhoenixPredicateDecomposer is not created because
- // it's not called method of StorageHandler.decomposePredicate.
-
- private static final Map<String, List<PhoenixPredicateDecomposer>> PREDICATE_DECOMPOSER_MAP =
- Maps.newConcurrentMap();
-
- public static PhoenixPredicateDecomposer createPredicateDecomposer(String predicateKey,
- List<String>
- columnNameList) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
- if (predicateDecomposerList == null) {
- predicateDecomposerList = Lists.newArrayList();
- PREDICATE_DECOMPOSER_MAP.put(predicateKey, predicateDecomposerList);
- }
-
- PhoenixPredicateDecomposer predicateDecomposer = new PhoenixPredicateDecomposer
- (columnNameList);
- predicateDecomposerList.add(predicateDecomposer);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" +
- predicateKey + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- public static PhoenixPredicateDecomposer getPredicateDecomposer(String predicateKey) {
- List<PhoenixPredicateDecomposer> predicateDecomposerList = PREDICATE_DECOMPOSER_MAP.get
- (predicateKey);
-
- PhoenixPredicateDecomposer predicateDecomposer = null;
- if (predicateDecomposerList != null && predicateDecomposerList.size() > 0) {
- predicateDecomposer = predicateDecomposerList.remove(0);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Predicate-decomposer : " + PREDICATE_DECOMPOSER_MAP + " [" + predicateKey
- + "] : " + predicateDecomposer);
- }
-
- return predicateDecomposer;
- }
-
- private PhoenixPredicateDecomposerManager() {
- }
-}