You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/05/09 15:35:39 UTC
hive git commit: HIVE-10865: Beeline needs to support DELIMITER
command (Sahil Takiar, reviewed by Sergio Pena, Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master 78e29fc70 -> 067d953bf
HIVE-10865: Beeline needs to support DELIMITER command (Sahil Takiar, reviewed by Sergio Pena, Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/067d953b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/067d953b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/067d953b
Branch: refs/heads/master
Commit: 067d953bfe45d61d3dff77ef12d9af08f1a45ae6
Parents: 78e29fc
Author: Sahil Takiar <ta...@gmail.com>
Authored: Tue May 9 10:34:56 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Tue May 9 10:34:56 2017 -0500
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 8 +-
.../org/apache/hive/beeline/BeeLineOpts.java | 11 +++
.../java/org/apache/hive/beeline/Commands.java | 94 ++++++++++----------
beeline/src/main/resources/BeeLine.properties | 3 +
.../hive/beeline/TestBeeLineWithArgs.java | 42 +++++++++
5 files changed, 108 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/067d953b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
index a589f33..444b3cf 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java
@@ -281,6 +281,8 @@ public class BeeLine implements Closeable {
new ReflectiveCommandHandler(this, new String[]{"addlocaldriverjar"},
null),
new ReflectiveCommandHandler(this, new String[]{"addlocaldrivername"},
+ null),
+ new ReflectiveCommandHandler(this, new String[]{"delimiter"},
null)
};
@@ -1357,7 +1359,7 @@ public class BeeLine implements Closeable {
return false;
}
- return !trimmed.endsWith(";");
+ return !trimmed.endsWith(getOpts().getDelimiter());
}
/**
@@ -1408,7 +1410,7 @@ public class BeeLine implements Closeable {
// we're continuing an existing command
cmd.append("\n");
cmd.append(scriptLine);
- if (trimmedLine.endsWith(";")) {
+ if (trimmedLine.endsWith(getOpts().getDelimiter())) {
// this command has terminated
cmds.add(cmd.toString());
cmd = null;
@@ -1429,7 +1431,7 @@ public class BeeLine implements Closeable {
// ### REVIEW: oops, somebody left the last command
// unterminated; should we fix it for them or complain?
// For now be nice and fix it.
- cmd.append(";");
+ cmd.append(getOpts().getDelimiter());
cmds.add(cmd.toString());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/067d953b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
index f85d8a3..3ebbc9a 100644
--- a/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
+++ b/beeline/src/java/org/apache/hive/beeline/BeeLineOpts.java
@@ -61,6 +61,7 @@ class BeeLineOpts implements Completer {
public static final char DEFAULT_DELIMITER_FOR_DSV = '|';
public static final int DEFAULT_MAX_COLUMN_WIDTH = 50;
public static final int DEFAULT_INCREMENTAL_BUFFER_ROWS = 1000;
+ public static final String DEFAULT_DELIMITER = ";";
public static final String URL_ENV_PREFIX = "BEELINE_URL_";
@@ -116,6 +117,8 @@ class BeeLineOpts implements Completer {
private TreeSet<String> cachedPropertyNameSet = null;
+ private String delimiter = DEFAULT_DELIMITER;
+
@Retention(RetentionPolicy.RUNTIME)
public @interface Ignore {
// marker annotations for functions that Reflector should ignore / pretend it does not exist
@@ -659,6 +662,14 @@ class BeeLineOpts implements Completer {
this.lastConnectedUrl = lastConnectedUrl;
}
+ public String getDelimiter() {
+ return this.delimiter;
+ }
+
+ public void setDelimiter(String delimiter) {
+ this.delimiter = delimiter;
+ }
+
@Ignore
public static Env getEnv(){
return env;
http://git-wip-us.apache.org/repos/asf/hive/blob/067d953b/beeline/src/java/org/apache/hive/beeline/Commands.java
----------------------------------------------------------------------
diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java
index 407e018..da896a7 100644
--- a/beeline/src/java/org/apache/hive/beeline/Commands.java
+++ b/beeline/src/java/org/apache/hive/beeline/Commands.java
@@ -67,6 +67,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hive.jdbc.logs.InPlaceUpdateStream;
public class Commands {
+
private final BeeLine beeLine;
private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000;
private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000;
@@ -297,7 +298,7 @@ public class Commands {
try {
while (rs.next()) {
cmds.add("DROP TABLE "
- + rs.getString("TABLE_NAME") + ";");
+ + rs.getString("TABLE_NAME") + beeLine.getOpts().getDelimiter());
}
} finally {
try {
@@ -892,7 +893,7 @@ public class Commands {
}
extra = reader.readLine();
}
- String[] cmds = lines.split(";");
+ String[] cmds = lines.split(beeLine.getOpts().getDelimiter());
for (String c : cmds) {
c = c.trim();
if (!executeInternal(c, false)) {
@@ -1131,7 +1132,7 @@ public class Commands {
//assumes line would never be null when this method is called
private boolean isMultiLine(String line) {
line = line.trim();
- if (line.endsWith(";") || beeLine.isComment(line)) {
+ if (line.endsWith(beeLine.getOpts().getDelimiter()) || beeLine.isComment(line)) {
return false;
}
// handles the case like line = show tables; --test comment
@@ -1202,7 +1203,7 @@ public class Commands {
// the continuation lines! This is logged as sf.net
// bug 879518.
- // use multiple lines for statements not terminated by ";"
+ // use multiple lines for statements not terminated by the delimiter
try {
line = handleMultiLineCmd(line);
} catch (Exception e) {
@@ -1224,8 +1225,8 @@ public class Commands {
/**
* Helper method to parse input from Beeline and convert it to a {@link List} of commands that
- * can be executed. This method contains logic for handling semicolons that are placed within
- * quotations. It iterates through each character in the line and checks to see if it is a ;, ',
+ * can be executed. This method contains logic for handling delimiters that are placed within
+ * quotations. It iterates through each character in the line and checks to see if it is the delimiter, ',
* or "
*/
private List<String> getCmdList(String line, boolean entireLineAsCommand) {
@@ -1241,55 +1242,50 @@ public class Commands {
// Marker to track if there is starting single quote without an ending double quote
boolean hasUnterminatedSingleQuote = false;
- // Index of the last seen semicolon in the given line
- int lastSemiColonIndex = 0;
- char[] lineChars = line.toCharArray();
+ // Index of the last seen delimiter in the given line
+ int lastDelimiterIndex = 0;
// Marker to track if the previous character was an escape character
boolean wasPrevEscape = false;
int index = 0;
- // Iterate through the line and invoke the addCmdPart method whenever a semicolon is seen that is not inside a
+ // Iterate through the line and invoke the addCmdPart method whenever the delimiter is seen that is not inside a
// quoted string
- for (; index < lineChars.length; index++) {
- switch (lineChars[index]) {
- case '\'':
- // If a single quote is seen and the index is not inside a double quoted string and the previous character
- // was not an escape, then update the hasUnterminatedSingleQuote flag
- if (!hasUnterminatedDoubleQuote && !wasPrevEscape) {
- hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote;
- }
- wasPrevEscape = false;
- break;
- case '\"':
- // If a double quote is seen and the index is not inside a single quoted string and the previous character
- // was not an escape, then update the hasUnterminatedDoubleQuote flag
- if (!hasUnterminatedSingleQuote && !wasPrevEscape) {
- hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote;
- }
- wasPrevEscape = false;
- break;
- case ';':
- // If a semicolon is seen, and the line isn't inside a quoted string, then treat
- // line[lastSemiColonIndex] to line[index] as a single command
- if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) {
- addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index));
- lastSemiColonIndex = index + 1;
- }
- wasPrevEscape = false;
- break;
- case '\\':
- wasPrevEscape = !wasPrevEscape;
- break;
- default:
- wasPrevEscape = false;
- break;
+ for (; index < line.length();) {
+ if (line.startsWith("\'", index)) {
+ // If a single quote is seen and the index is not inside a double quoted string and the previous character
+ // was not an escape, then update the hasUnterminatedSingleQuote flag
+ if (!hasUnterminatedDoubleQuote && !wasPrevEscape) {
+ hasUnterminatedSingleQuote = !hasUnterminatedSingleQuote;
+ }
+ wasPrevEscape = false;
+ index++;
+ } else if (line.startsWith("\"", index)) {
+ // If a double quote is seen and the index is not inside a single quoted string and the previous character
+ // was not an escape, then update the hasUnterminatedDoubleQuote flag
+ if (!hasUnterminatedSingleQuote && !wasPrevEscape) {
+ hasUnterminatedDoubleQuote = !hasUnterminatedDoubleQuote;
+ }
+ wasPrevEscape = false;
+ index++;
+ } else if (line.startsWith(beeLine.getOpts().getDelimiter(), index)) {
+ // If the delimiter is seen, and the line isn't inside a quoted string, then treat
+ // line[lastDelimiterIndex] to line[index] as a single command
+ if (!hasUnterminatedDoubleQuote && !hasUnterminatedSingleQuote) {
+ addCmdPart(cmdList, command, line.substring(lastDelimiterIndex, index));
+ lastDelimiterIndex = index + beeLine.getOpts().getDelimiter().length();
+ }
+ wasPrevEscape = false;
+ index += beeLine.getOpts().getDelimiter().length();
+ } else {
+ wasPrevEscape = line.startsWith("\\", index) && !wasPrevEscape;
+ index++;
}
}
- // If the line doesn't end with a ; or if the line is empty, add the cmd part
- if (lastSemiColonIndex != index || lineChars.length == 0) {
- addCmdPart(cmdList, command, line.substring(lastSemiColonIndex, index));
+ // If the line doesn't end with the delimiter or if the line is empty, add the cmd part
+ if (lastDelimiterIndex != index || line.length() == 0) {
+ addCmdPart(cmdList, command, line.substring(lastDelimiterIndex, index));
}
}
return cmdList;
@@ -1301,7 +1297,7 @@ public class Commands {
*/
private void addCmdPart(List<String> cmdList, StringBuilder command, String cmdpart) {
if (cmdpart.endsWith("\\")) {
- command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";");
+ command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(beeLine.getOpts().getDelimiter());
return;
} else {
command.append(cmdpart);
@@ -1949,4 +1945,8 @@ public class Commands {
breader.close();
return true;
}
+
+ public boolean delimiter(String line) {
+ return set("set " + line);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/067d953b/beeline/src/main/resources/BeeLine.properties
----------------------------------------------------------------------
diff --git a/beeline/src/main/resources/BeeLine.properties b/beeline/src/main/resources/BeeLine.properties
index 7011221..3b8e3e6 100644
--- a/beeline/src/main/resources/BeeLine.properties
+++ b/beeline/src/main/resources/BeeLine.properties
@@ -73,6 +73,7 @@ help-delimiterForDSV: Set the delimiter for dsv output format
help-nullemptystring: Set to true to get historic behavior of printing null as empty string. Default is false.
help-addlocaldriverjar: Add driver jar file in the beeline client side.
help-addlocaldrivername: Add driver name that needs to be supported in the beeline client side.
+help-delimiter: Sets the query delimiter, defaults to ;
jline-missing: The JLine jar was not found. Please ensure it is installed.
@@ -203,6 +204,8 @@ cmd-usage: Usage: java org.apache.hive.cli.beeline.BeeLine \n \
\ --isolation=LEVEL set the transaction isolation level\n \
\ --nullemptystring=[true/false] set to true to get historic behavior of printing null as empty string\n \
\ --maxHistoryRows=MAXHISTORYROWS The maximum number of rows to store beeline history.\n \
+\ --delimiter=DELIMITER set the query delimiter; multi-char delimiters are allowed, but quotation\n \
+\ marks, slashes, and -- are not allowed; defaults to ;\n \
\ --help display this message\n \
\n \
\ Example:\n \
http://git-wip-us.apache.org/repos/asf/hive/blob/067d953b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
index 75f46ec..d90165b 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java
@@ -1082,4 +1082,46 @@ public class TestBeeLineWithArgs {
argList.add("--outputformat=tsv2");
testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
}
+
+ @Test
+ public void testCustomDelimiter() throws Throwable {
+ String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$";
+ final String EXPECTED_PATTERN = "hello\thello\thello";
+ List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+ argList.add("--delimiter=$");
+ argList.add("--outputformat=tsv2");
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
+ }
+
+ @Test
+ public void testCustomMultiCharDelimiter() throws Throwable {
+ String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$$";
+ final String EXPECTED_PATTERN = "hello\thello\thello";
+ List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+ argList.add("--delimiter=$$");
+ argList.add("--outputformat=tsv2");
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
+ }
+
+ @Test
+ public void testCustomDelimiterWithMultiQuery() throws Throwable {
+ String SCRIPT_TEXT = "select 'hello', 'hello', 'hello'$select 'world', 'world', 'world'$";
+ final String EXPECTED_PATTERN1 = "hello\thello\thello";
+ final String EXPECTED_PATTERN2 = "world\tworld\tworld";
+ List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+ argList.add("--delimiter=$");
+ argList.add("--outputformat=tsv2");
+ List<Tuple<String>> expectedMatches = Arrays.asList(new Tuple<>(EXPECTED_PATTERN1, true),
+ new Tuple<>(EXPECTED_PATTERN2, true));
+ testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, expectedMatches);
+ }
+
+ @Test
+ public void testCustomDelimiterBeelineCmd() throws Throwable {
+ String SCRIPT_TEXT = "!delimiter $\n select 'hello', 'hello', 'hello'$";
+ final String EXPECTED_PATTERN = "hello\thello\thello";
+ List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL());
+ argList.add("--outputformat=tsv2");
+ testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true);
+ }
}