You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2014/08/12 20:06:10 UTC
svn commit: r1617547 - in /hive/trunk: ./ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
itests/hive-unit/src/test/java/org/apache/hive/beeline/
Author: szehon
Date: Tue Aug 12 18:06:09 2014
New Revision: 1617547
URL: http://svn.apache.org/r1617547
Log:
HIVE-7390 : Make quote character optional and configurable in BeeLine CSV/TSV output (Ferdinand Xu via Szehon)
Modified:
hive/trunk/beeline/pom.xml
hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
hive/trunk/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
hive/trunk/beeline/src/main/resources/BeeLine.properties
hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
hive/trunk/pom.xml
Modified: hive/trunk/beeline/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/pom.xml?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/beeline/pom.xml (original)
+++ hive/trunk/beeline/pom.xml Tue Aug 12 18:06:09 2014
@@ -80,6 +80,11 @@
<artifactId>libthrift</artifactId>
<version>${libthrift.version}</version>
</dependency>
+ <dependency>
+ <groupId>net.sf.supercsv</groupId>
+ <artifactId>super-csv</artifactId>
+ <version>${super-csv.version}</version>
+ </dependency>
<!-- test intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue Aug 12 18:06:09 2014
@@ -79,13 +79,13 @@ import jline.ConsoleReader;
import jline.FileNameCompletor;
import jline.History;
import jline.SimpleCompletor;
-import org.apache.hadoop.io.IOUtils;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.io.IOUtils;
/**
@@ -148,6 +148,7 @@ public class BeeLine implements Closeabl
"table", new TableOutputFormat(this),
"csv", new SeparatedValuesOutputFormat(this, ','),
"tsv", new SeparatedValuesOutputFormat(this, '\t'),
+ "dsv", new SeparatedValuesOutputFormat(this, BeeLineOpts.DEFAULT_DELIMITER_FOR_DSV),
"xmlattr", new XMLAttributeOutputFormat(this),
"xmlelements", new XMLElementOutputFormat(this),
});
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java Tue Aug 12 18:06:09 2014
@@ -53,6 +53,7 @@ class BeeLineOpts implements Completor {
public static final String PROPERTY_NAME_EXIT =
PROPERTY_PREFIX + "system.exit";
public static final String DEFAULT_NULL_STRING = "NULL";
+ public static final char DEFAULT_DELIMITER_FOR_DSV = '|';
private final BeeLine beeLine;
private boolean autosave = false;
@@ -90,7 +91,7 @@ class BeeLineOpts implements Completor {
private String scriptFile = null;
private String initFile = null;
private String authType = null;
-
+ private char delimiterForDSV = DEFAULT_DELIMITER_FOR_DSV;
private Map<String, String> hiveVariables = new HashMap<String, String>();
private Map<String, String> hiveConfVariables = new HashMap<String, String>();
@@ -500,5 +501,13 @@ class BeeLineOpts implements Completor {
public void setTruncateTable(boolean truncateTable) {
this.truncateTable = truncateTable;
}
+
+ public char getDelimiterForDSV() {
+ return delimiterForDSV;
+ }
+
+ public void setDelimiterForDSV(char delimiterForDSV) {
+ this.delimiterForDSV = delimiterForDSV;
+ }
}
Modified: hive/trunk/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java (original)
+++ hive/trunk/beeline/src/java/org/apache/hive/beeline/SeparatedValuesOutputFormat.java Tue Aug 12 18:06:09 2014
@@ -22,25 +22,43 @@
*/
package org.apache.hive.beeline;
+import java.io.IOException;
+import java.io.StringWriter;
+
+import org.apache.hadoop.io.IOUtils;
+import org.supercsv.io.CsvListWriter;
+import org.supercsv.prefs.CsvPreference;
+
/**
* OutputFormat for values separated by a delimiter.
- *
- * <strong>TODO</strong>: Handle character escaping
- *
*/
class SeparatedValuesOutputFormat implements OutputFormat {
/**
*
*/
private final BeeLine beeLine;
- private char separator;
+ private CsvPreference csvPreference;
- public SeparatedValuesOutputFormat(BeeLine beeLine, char separator) {
+ SeparatedValuesOutputFormat(BeeLine beeLine, char separator) {
this.beeLine = beeLine;
- setSeparator(separator);
+ csvPreference = new CsvPreference.Builder('"', separator, "").build();
+ }
+
+ private void updateCsvPreference() {
+ if (beeLine.getOpts().getOutputFormat().equals("dsv")) {
+ // check whether delimiter changed by user
+ char curDel = (char) csvPreference.getDelimiterChar();
+ char newDel = beeLine.getOpts().getDelimiterForDSV();
+ // if delimiter changed, rebuild the csv preference
+ if (newDel != curDel) {
+ csvPreference = new CsvPreference.Builder('"', newDel, "").build();
+ }
+ }
}
public int print(Rows rows) {
+ updateCsvPreference();
+
int count = 0;
while (rows.hasNext()) {
printRow(rows, (Rows.Row) rows.next());
@@ -49,23 +67,24 @@ class SeparatedValuesOutputFormat implem
return count - 1; // sans header row
}
- public void printRow(Rows rows, Rows.Row row) {
- String[] vals = row.values;
- StringBuilder buf = new StringBuilder();
- for (int i = 0; i < vals.length; i++) {
- buf.append(buf.length() == 0 ? "" : "" + getSeparator())
- .append('\'')
- .append(vals[i] == null ? "" : vals[i])
- .append('\'');
+ private String getFormattedStr(String[] vals) {
+ StringWriter strWriter = new StringWriter();
+ CsvListWriter writer = new CsvListWriter(strWriter, csvPreference);
+ if (vals.length > 0) {
+ try {
+ writer.write(vals);
+ } catch (IOException e) {
+ beeLine.error(e);
+ } finally {
+ IOUtils.closeStream(writer);
+ }
}
- beeLine.output(buf.toString());
- }
-
- public void setSeparator(char separator) {
- this.separator = separator;
+ return strWriter.toString();
}
- public char getSeparator() {
- return this.separator;
+ public void printRow(Rows rows, Rows.Row row) {
+ String[] vals = row.values;
+ String formattedStr = getFormattedStr(vals);
+ beeLine.output(formattedStr);
}
}
Modified: hive/trunk/beeline/src/main/resources/BeeLine.properties
URL: http://svn.apache.org/viewvc/hive/trunk/beeline/src/main/resources/BeeLine.properties?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/beeline/src/main/resources/BeeLine.properties (original)
+++ hive/trunk/beeline/src/main/resources/BeeLine.properties Tue Aug 12 18:06:09 2014
@@ -68,7 +68,8 @@ help-procedures: List all the procedures
help-tables: List all the tables in the database
help-columns: List all the columns for the specified table
help-properties: Connect to the database specified in the properties file(s)
-help-outputformat: Set the output format for displaying results (table,vertical,csv,tsv,xmlattrs,xmlelements)
+help-outputformat: Set the output format for displaying results (table,vertical,csv,dsv,tsv,xmlattrs,xmlelements)
+help-delimiterForDSV: Set the delimiter for dsv output format
help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false.
jline-missing: The JLine jar was not found. Please ensure it is installed.
@@ -166,8 +167,9 @@ cmd-usage: Usage: java org.apache.hive.c
\ --maxColumnWidth=MAXCOLWIDTH the maximum width to use when displaying columns\n \
\ --silent=[true/false] be more silent\n \
\ --autosave=[true/false] automatically save preferences\n \
-\ --outputformat=[table/vertical/csv/tsv] format mode for result display\n \
+\ --outputformat=[table/vertical/csv/tsv/dsv] format mode for result display\n \
\ --truncateTable=[true/false] truncate table column when it exceeds length\n \
+\ --delimiterForDSV=DELIMITER specify the delimiter for delimiter-separated values output format (default: |)\n \
\ --isolation=LEVEL set the transaction isolation level\n \
\ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \
\ --help display this message
Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java Tue Aug 12 18:06:09 2014
@@ -362,7 +362,7 @@ public class TestBeeLineWithArgs {
final String TEST_NAME = "testNullNonDefault";
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
"!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
- final String EXPECTED_PATTERN = "'abc','','def'";
+ final String EXPECTED_PATTERN = "abc,,def";
List<String> argList = getBaseArgs(JDBC_URL);
argList.add("--outputformat=csv");
@@ -382,7 +382,7 @@ public class TestBeeLineWithArgs {
final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" +
"select 'abc',null,'def' from " + tableName + " limit 1 ;\n";
//final String EXPECTED_PATTERN = "| abc | | def |";
- final String EXPECTED_PATTERN = "'abc','','def'";
+ final String EXPECTED_PATTERN = "abc,,def";
List<String> argList = getBaseArgs(JDBC_URL);
argList.add("--nullemptystring=true");
Modified: hive/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hive/trunk/pom.xml?rev=1617547&r1=1617546&r2=1617547&view=diff
==============================================================================
--- hive/trunk/pom.xml (original)
+++ hive/trunk/pom.xml Tue Aug 12 18:06:09 2014
@@ -146,6 +146,7 @@
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.5</slf4j.version>
<ST4.version>4.0.4</ST4.version>
+ <super-csv.version>2.2.0</super-csv.version>
<tez.version>0.4.0-incubating</tez.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
<snappy.version>0.2</snappy.version>