You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dm...@apache.org on 2019/10/21 13:27:31 UTC
[hive] branch master updated: HIVE-21246: Un-bury
DelimitedJSONSerDe from PlanUtils.java (David Mollitor,
reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
dmollitor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c9850b4 HIVE-21246: Un-bury DelimitedJSONSerDe from PlanUtils.java (David Mollitor, reviewed by Laszlo Bodor)
c9850b4 is described below
commit c9850b443de4e38903ae4c91a9619a71b2f3db33
Author: David Mollitor <dm...@apache.org>
AuthorDate: Mon Oct 21 09:26:06 2019 -0400
HIVE-21246: Un-bury DelimitedJSONSerDe from PlanUtils.java (David Mollitor, reviewed by Laszlo Bodor)
---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 ++++--
.../org/apache/hadoop/hive/ql/plan/PlanUtils.java | 34 +++++-----------------
2 files changed, 15 insertions(+), 28 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 281025f..30d3791 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -257,6 +257,7 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFInline;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
import org.apache.hadoop.hive.ql.util.ResourceDownloader;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.NoOpFetchFormatter;
@@ -4010,9 +4011,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
.getChild(inputSerDeNum))).getChild(0), inpColumns.toString(),
inpColumnTypes.toString());
} else {
- inInfo = PlanUtils.getTableDesc(serde, Integer
+ // It is not a very clean way, and should be modified later - due to
+ // compatibility reasons, user sees the results as JSON for custom
+ // scripts and has no way for specifying that. Right now, it is
+ // hard-coded to DelimitedJSONSerDe
+ inInfo = PlanUtils.getTableDesc(DelimitedJSONSerDe.class, Integer
.toString(fieldSeparator), inpColumns.toString(), inpColumnTypes
- .toString(), false, true);
+ .toString(), false);
}
if (trfm.getChild(outputSerDeNum).getChildCount() > 0) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
index 563d053..e20f695 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
@@ -67,7 +67,6 @@ import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
-import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
@@ -234,26 +233,18 @@ public final class PlanUtils {
columnTypes, lastColumnTakesRestOfTheLine);
}
- public static TableDesc getTableDesc(
- Class<? extends Deserializer> serdeClass, String separatorCode,
- String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine) {
- return getTableDesc(serdeClass, separatorCode, columns, columnTypes,
- lastColumnTakesRestOfTheLine, false);
- }
-
- public static TableDesc getTableDesc(
- Class<? extends Deserializer> serdeClass, String separatorCode,
- String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useDelimitedJSON) {
+ public static TableDesc getTableDesc(Class<? extends Deserializer> serdeClass,
+ String separatorCode, String columns, String columnTypes,
+ boolean lastColumnTakesRestOfTheLine) {
return getTableDesc(serdeClass, separatorCode, columns, columnTypes,
- lastColumnTakesRestOfTheLine, useDelimitedJSON, "TextFile");
+ lastColumnTakesRestOfTheLine, "TextFile");
}
public static TableDesc getTableDesc(
Class<? extends Deserializer> serdeClass, String separatorCode,
String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useDelimitedJSON, String fileFormat) {
+ String fileFormat) {
Properties properties = Utilities.makeProperties(
serdeConstants.SERIALIZATION_FORMAT, separatorCode, serdeConstants.LIST_COLUMNS,
@@ -272,15 +263,6 @@ public final class PlanUtils {
"true");
}
- // It is not a very clean way, and should be modified later - due to
- // compatibility reasons,
- // user sees the results as json for custom scripts and has no way for
- // specifying that.
- // Right now, it is hard-coded in the code
- if (useDelimitedJSON) {
- serdeClass = DelimitedJSONSerDe.class;
- }
-
Class inputFormat, outputFormat;
// get the input & output file formats
if ("HiveSequenceFile".equalsIgnoreCase(fileFormat)) {
@@ -308,7 +290,7 @@ public final class PlanUtils {
public static TableDesc getDefaultQueryOutputTableDesc(String cols, String colTypes,
String fileFormat, Class<? extends Deserializer> serdeClass) {
TableDesc tblDesc =
- getTableDesc(serdeClass, "" + Utilities.ctrlaCode, cols, colTypes, false, false, fileFormat);
+ getTableDesc(serdeClass, "" + Utilities.ctrlaCode, cols, colTypes, false, fileFormat);
// enable escaping
tblDesc.getProperties().setProperty(serdeConstants.ESCAPE_CHAR, "\\");
tblDesc.getProperties().setProperty(serdeConstants.SERIALIZATION_ESCAPE_CRLF, "true");
@@ -351,7 +333,7 @@ public final class PlanUtils {
}
ret = getTableDesc(serdeClass, separatorCode, columns, columnTypes,
- lastColumnTakesRestOfTheLine, false);
+ lastColumnTakesRestOfTheLine);
// set other table properties
Properties properties = ret.getProperties();
@@ -448,7 +430,7 @@ public final class PlanUtils {
}
ret = getTableDesc(serdeClass, separatorCode, columns, columnTypes,
- lastColumnTakesRestOfTheLine, false);
+ lastColumnTakesRestOfTheLine);
// set other table properties
Properties properties = ret.getProperties();