You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/09/18 13:54:31 UTC
svn commit: r696641 - in /hadoop/core/trunk: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/JobHistory.java
src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java
Author: ddas
Date: Thu Sep 18 04:54:31 2008
New Revision: 696641
URL: http://svn.apache.org/viewvc?rev=696641&view=rev
Log:
HADOOP-2403. Escapes some special characters before logging to history files. Contributed by Amareshwari Sriramadasu.
Added:
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696641&r1=696640&r2=696641&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 18 04:54:31 2008
@@ -619,6 +619,9 @@
HADOOP-4195. Close compressor before returning to codec pool.
(acmurthy via omalley)
+ HADOOP-2403. Escapes some special characters before logging to
+ history files. (Amareshwari Sriramadasu via ddas)
+
Release 0.18.1 - 2008-09-17
IMPROVEMENTS
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=696641&r1=696640&r2=696641&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Thu Sep 18 04:54:31 2008
@@ -68,8 +68,12 @@
public static final Log LOG = LogFactory.getLog(JobHistory.class);
private static final String DELIMITER = " ";
private static final String LINE_DELIMITER = ".";
+ private static final char LINE_DELIMITER_CHAR = '.';
+ private static final char[] charsToEscape = new char[] {'"', '=',
+ LINE_DELIMITER_CHAR};
private static final String KEY = "(\\w+)";
- private static final String VALUE = "[[^\"]?]+"; // anything but a " in ""
+ // value is any character other than quote, but escaped quotes can be there
+ private static final String VALUE = "[^\"\\\\]*(?:\\\\.[^\"\\\\]*)*";
private static final Pattern pattern = Pattern.compile(KEY + "=" + "\"" + VALUE + "\"");
@@ -177,7 +181,10 @@
StringBuffer buf = new StringBuffer();
while ((line = reader.readLine())!= null){
buf.append(line);
- if (!line.trim().endsWith("\"" + DELIMITER + LINE_DELIMITER)){
+ if (!line.trim().endsWith(LINE_DELIMITER) ||
+ line.trim().endsWith(StringUtils.escapeString(LINE_DELIMITER,
+ StringUtils.ESCAPE_CHAR, LINE_DELIMITER_CHAR))) {
+ buf.append("\n");
continue;
}
parseLine(buf.toString(), l);
@@ -204,9 +211,11 @@
while(matcher.find()){
String tuple = matcher.group(0);
- String []parts = tuple.split("=");
-
- parseBuffer.put(Keys.valueOf(parts[0]), parts[1].substring(1, parts[1].length() -1));
+ String []parts = StringUtils.split(tuple, StringUtils.ESCAPE_CHAR, '=');
+ String value = parts[1].substring(1, parts[1].length() -1);
+ value = StringUtils.unEscapeString(value, StringUtils.ESCAPE_CHAR,
+ charsToEscape);
+ parseBuffer.put(Keys.valueOf(parts[0]), value);
}
l.handle(RecordTypes.valueOf(recType), parseBuffer);
@@ -224,6 +233,8 @@
static void log(PrintWriter out, RecordTypes recordType, Keys key,
String value){
+ value = StringUtils.escapeString(value, StringUtils.ESCAPE_CHAR,
+ charsToEscape);
out.println(recordType.name() + DELIMITER + key + "=\"" + value + "\""
+ DELIMITER + LINE_DELIMITER);
}
@@ -243,6 +254,8 @@
for(int i =0; i< keys.length; i++){
buf.append(keys[i]);
buf.append("=\"");
+ values[i] = StringUtils.escapeString(values[i],
+ StringUtils.ESCAPE_CHAR, charsToEscape);
buf.append(values[i]);
buf.append("\"");
buf.append(DELIMITER);
Added: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java?rev=696641&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java (added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestJobHistoryParsing.java Thu Sep 18 04:54:31 2008
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobHistory.*;
+
+import junit.framework.TestCase;
+
+public class TestJobHistoryParsing extends TestCase {
+ ArrayList<PrintWriter> historyWriter = new ArrayList<PrintWriter>();
+
+ /**
+ * Listener for a test history log file, it populates JobHistory.JobInfo
+ * object with data from log file.
+ */
+ static class TestListener implements Listener {
+ JobInfo job;
+
+ TestListener(JobInfo job) {
+ this.job = job;
+ }
+ // JobHistory.Listener implementation
+ public void handle(RecordTypes recType,
+ Map<JobHistory.Keys, String> values)
+ throws IOException {
+ if (recType == JobHistory.RecordTypes.Job) {
+ job.handle(values);
+ }
+ }
+ }
+
+ public void testHistoryParsing() throws IOException {
+ // open a test history file
+ Path historyDir = new Path(System.getProperty("test.build.data", "."),
+ "history");
+ FileSystem fs = FileSystem.getLocal(new JobConf());
+ if (!fs.mkdirs(historyDir)) {
+ fail("Failed to create history directory");
+ }
+ Path historyLog = new Path(historyDir, "testlog");
+ PrintWriter out = new PrintWriter(fs.create(historyLog));
+ historyWriter.add(out);
+ // log keys and values into history
+ String value1 = "Value has equal=to, \"quotes\" and spaces in it";
+ String value2 = "Value has \n new line \n and " +
+ "dot followed by new line .\n in it ";
+ String value3 = "Value has characters: " +
+ "`1234567890-=qwertyuiop[]\\asdfghjkl;'zxcvbnm,./" +
+ "~!@#$%^&*()_+QWERTYUIOP{}|ASDFGHJKL:\"'ZXCVBNM<>?" +
+ "\t\b\n\f\"\n in it";
+ String value4 = "Value ends with escape\\";
+ String value5 = "Value ends with \\\" \\.\n";
+ JobHistory.log(historyWriter, RecordTypes.Job,
+ new JobHistory.Keys[] {Keys.JOBTRACKERID,
+ Keys.TRACKER_NAME,
+ Keys.JOBNAME,
+ Keys.JOBCONF,
+ Keys.USER},
+ new String[] {value1, value2, value3, value4, value5});
+ // close history file
+ out.close();
+ historyWriter.remove(out);
+
+ // parse history
+ String jobId = "job_200809171136_0001"; // random jobid for tesing
+ JobHistory.JobInfo job = new JobHistory.JobInfo(jobId);
+ JobHistory.parseHistoryFromFS(historyLog.toString(),
+ new TestListener(job), fs);
+ // validate keys and values
+ assertEquals(value1, job.get(Keys.JOBTRACKERID));
+ assertEquals(value2, job.get(Keys.TRACKER_NAME));
+ assertEquals(value3, job.get(Keys.JOBNAME));
+ assertEquals(value4, job.get(Keys.JOBCONF));
+ assertEquals(value5, job.get(Keys.USER));
+ }
+}