You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2017/01/31 17:36:45 UTC
svn commit: r1781129 - in /pig/trunk: ./ ivy/ src/org/apache/pig/
src/org/apache/pig/tools/grunt/ src/org/apache/pig/tools/parameters/
src/org/apache/pig/tools/pigscript/parser/ test/org/apache/pig/test/
Author: daijy
Date: Tue Jan 31 17:36:45 2017
New Revision: 1781129
URL: http://svn.apache.org/viewvc?rev=1781129&view=rev
Log:
PIG-3851: Upgrade jline to 2.11
Added:
pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/ivy/libraries.properties
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java
pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java
pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
pig/trunk/test/org/apache/pig/test/TestGrunt.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 31 17:36:45 2017
@@ -34,6 +34,8 @@ PIG-5067: Revisit union on numeric type
IMPROVEMENTS
+PIG-3851: Upgrade jline to 2.11 (daijy)
+
PIG-4963: Add a Bloom join (rohini)
PIG-3938: Add LoadCaster to EvalFunc (knoguchi)
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Tue Jan 31 17:36:45 2017
@@ -56,7 +56,7 @@ jdeb.version=0.8
jdiff.version=1.0.9
jettison.version=1.3.4
jetty.version=6.1.26
-jline.version=1.0
+jline.version=2.11
joda-time.version=2.9.3
jopt.version=4.1
json-simple.version=1.1
Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Jan 31 17:36:45 2017
@@ -27,7 +27,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.StringReader;
import java.net.URL;
@@ -45,9 +44,8 @@ import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
-import jline.ConsoleReader;
-import jline.ConsoleReaderInputStream;
-import jline.History;
+import jline.console.ConsoleReader;
+import jline.console.history.FileHistory;
import org.antlr.runtime.RecognitionException;
import org.apache.commons.logging.Log;
@@ -77,6 +75,7 @@ import org.apache.pig.parser.DryRunGrunt
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.scripting.ScriptEngine.SupportedScriptLang;
import org.apache.pig.tools.cmdline.CmdLineParser;
+import org.apache.pig.tools.grunt.ConsoleReaderInputStream;
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
import org.apache.pig.tools.pigstats.PigStats;
@@ -551,12 +550,13 @@ public class Main {
}
// Interactive
mode = ExecMode.SHELL;
- //Reader is created by first loading "pig.load.default.statements" or .pigbootup file if available
- ConsoleReader reader = new ConsoleReader(Utils.getCompositeStream(System.in, properties), new OutputStreamWriter(System.out));
- reader.setDefaultPrompt("grunt> ");
+ //Reader is created by first loading "pig.load.default.statements" or .pigbootup file if available
+ ConsoleReader reader = new ConsoleReader(Utils.getCompositeStream(System.in, properties), System.out);
+ reader.setExpandEvents(false);
+ reader.setPrompt("grunt> ");
final String HISTORYFILE = ".pig_history";
String historyFile = System.getProperty("user.home") + File.separator + HISTORYFILE;
- reader.setHistory(new History(new File(historyFile)));
+ reader.setHistory(new FileHistory(new File(historyFile)));
ConsoleReaderInputStream inputStream = new ConsoleReaderInputStream(reader);
grunt = new Grunt(new BufferedReader(new InputStreamReader(inputStream)), pigContext);
grunt.setConsoleReader(reader);
Added: pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java?rev=1781129&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java (added)
+++ pig/trunk/src/org/apache/pig/tools/grunt/ConsoleReaderInputStream.java Tue Jan 31 17:36:45 2017
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tools.grunt;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.Enumeration;
+
+import jline.console.ConsoleReader;
+
+/** Borrowed from jline.console.internal.ConsoleReaderInputStream. However,
+ * we cannot use ConsoleReaderInputStream directly since:
+ * 1. ConsoleReaderInputStream is not public
+ * 2. ConsoleReaderInputStream has a bug which does not deal with UTF-8 correctly
+ */
+public class ConsoleReaderInputStream extends SequenceInputStream {
+ private static InputStream systemIn = System.in;
+
+ public static void setIn() throws IOException {
+ setIn(new ConsoleReader());
+ }
+
+ public static void setIn(final ConsoleReader reader) {
+ System.setIn(new ConsoleReaderInputStream(reader));
+ }
+
+ /**
+ * Restore the original {@link System#in} input stream.
+ */
+ public static void restoreIn() {
+ System.setIn(systemIn);
+ }
+
+ public ConsoleReaderInputStream(final ConsoleReader reader) {
+ super(new ConsoleEnumeration(reader));
+ }
+
+ private static class ConsoleEnumeration implements Enumeration {
+ private final ConsoleReader reader;
+ private ConsoleLineInputStream next = null;
+ private ConsoleLineInputStream prev = null;
+
+ public ConsoleEnumeration(final ConsoleReader reader) {
+ this.reader = reader;
+ }
+
+ public Object nextElement() {
+ if (next != null) {
+ InputStream n = next;
+ prev = next;
+ next = null;
+
+ return n;
+ }
+
+ return new ConsoleLineInputStream(reader);
+ }
+
+ public boolean hasMoreElements() {
+ // the last line was null
+ if ((prev != null) && (prev.wasNull == true)) {
+ return false;
+ }
+
+ if (next == null) {
+ next = (ConsoleLineInputStream) nextElement();
+ }
+
+ return next != null;
+ }
+ }
+
+ private static class ConsoleLineInputStream extends InputStream {
+ private final ConsoleReader reader;
+ private byte[] buffer = null;
+ private int index = 0;
+ private boolean eol = false;
+ protected boolean wasNull = false;
+
+ public ConsoleLineInputStream(final ConsoleReader reader) {
+ this.reader = reader;
+ }
+
+ public int read() throws IOException {
+ if (eol) {
+ return -1;
+ }
+
+ if (buffer == null) {
+ buffer = reader.readLine().getBytes();
+ }
+
+ if (buffer == null) {
+ wasNull = true;
+ return -1;
+ }
+
+ if (index >= buffer.length) {
+ eol = true;
+ return '\n'; // lines are ended with a newline
+ }
+
+ return buffer[index++];
+ }
+ }
+}
\ No newline at end of file
Modified: pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java Tue Jan 31 17:36:45 2017
@@ -20,7 +20,7 @@ package org.apache.pig.tools.grunt;
import java.io.BufferedReader;
import java.util.ArrayList;
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -52,8 +52,8 @@ public class Grunt
public void setConsoleReader(ConsoleReader c)
{
- c.addCompletor(new PigCompletorAliases(pig));
- c.addCompletor(new PigCompletor());
+ c.addCompleter(new PigCompletorAliases(pig));
+ c.addCompleter(new PigCompletor());
parser.setConsoleReader(c);
}
Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue Jan 31 17:36:45 2017
@@ -26,7 +26,6 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.Reader;
import java.io.StringReader;
@@ -42,8 +41,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import jline.ConsoleReader;
-import jline.ConsoleReaderInputStream;
+import jline.console.ConsoleReader;
import org.apache.commons.io.output.NullOutputStream;
import org.apache.commons.logging.Log;
@@ -264,7 +262,7 @@ public class GruntParser extends PigScri
public void prompt()
{
if (mInteractive) {
- mConsoleReader.setDefaultPrompt("grunt> ");
+ mConsoleReader.setPrompt("grunt> ");
}
}
@@ -533,7 +531,7 @@ public class GruntParser extends PigScri
cmds = cmds.replaceAll("\t"," ");
reader = new ConsoleReader(new ByteArrayInputStream(cmds.getBytes()),
- new OutputStreamWriter(System.out));
+ System.out);
reader.setHistory(mConsoleReader.getHistory());
InputStream in = new ConsoleReaderInputStream(reader);
inputReader = new BufferedReader(new InputStreamReader(in));
Modified: pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/PigCompletor.java Tue Jan 31 17:36:45 2017
@@ -33,9 +33,9 @@ import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import jline.Completor;
+import jline.console.completer.Completer;
-public class PigCompletor implements Completor {
+public class PigCompletor implements Completer {
private final Log log = LogFactory.getLog(getClass());
Set<String> candidates;
static final String AUTOCOMPLETE_FILENAME = "autocomplete";
Modified: pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/PigCompletorAliases.java Tue Jan 31 17:36:45 2017
@@ -26,12 +26,11 @@ import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.pig.PigServer;
-import jline.Completor;
+import jline.console.completer.Completer;
-public class PigCompletorAliases implements Completor {
+public class PigCompletorAliases implements Completer {
private final Log log = LogFactory.getLog(getClass());
Set<String> keywords;
PigServer pig;
Modified: pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Tue Jan 31 17:36:45 2017
@@ -42,7 +42,6 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.validator.BlackAndWhitelistFilter;
import org.apache.pig.validator.PigCommandFilter;
-import org.python.google.common.base.Preconditions;
public class PreprocessorContext {
Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Tue Jan 31 17:36:45 2017
@@ -37,7 +37,7 @@ import java.util.List;
import java.util.ArrayList;
import org.apache.pig.impl.util.StringUtils;
-import jline.ConsoleReader;
+import jline.console.ConsoleReader;
public abstract class PigScriptParser
{
@@ -218,7 +218,7 @@ TOKEN_MGR_DECLS : {
{
/*System.err.print(">> ");
System.err.flush();*/
- consoleReader.setDefaultPrompt(">> ");
+ consoleReader.setPrompt(">> ");
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=1781129&r1=1781128&r2=1781129&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestGrunt.java Tue Jan 31 17:36:45 2017
@@ -28,6 +28,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
+import java.io.FilenameFilter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintWriter;
@@ -1555,4 +1556,20 @@ public class TestGrunt {
}
assertTrue(found);
}
+
+ @Test
+ public void testGruntUtf8() throws Throwable {
+ String command = "mkdir 测试\n" +
+ "quit\n";
+ System.setProperty("jline.WindowsTerminal.directConsole", "false");
+ System.setIn(new ByteArrayInputStream(command.getBytes()));
+ org.apache.pig.PigRunner.run(new String[] {"-x", "local"}, null);
+ File[] partFiles = new File(".").listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.equals("测试");
+ }
+ });
+ assertEquals(partFiles.length, 1);
+ new File("测试").delete();
+ }
}