You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/12/14 00:01:35 UTC
[4/8] ACCUMULO-2010 Remove Closeable from interfaces
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
index 491b8ec..2caf7cd 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/UndefinedAnalyzer.java
@@ -49,96 +49,100 @@ import org.apache.hadoop.io.Text;
*
*/
public class UndefinedAnalyzer {
-
+
static class UndefinedNode {
-
+
public UndefinedNode(String undef2, String ref2) {
this.undef = undef2;
this.ref = ref2;
}
-
+
String undef;
String ref;
}
-
+
static class IngestInfo {
-
+
Map<String,TreeMap<Long,Long>> flushes = new HashMap<String,TreeMap<Long,Long>>();
-
+
public IngestInfo(String logDir) throws Exception {
File dir = new File(logDir);
File[] ingestLogs = dir.listFiles(new FilenameFilter() {
+ @Override
public boolean accept(java.io.File dir, String name) {
return name.endsWith("ingest.out");
}
});
-
+
for (File log : ingestLogs) {
parseLog(log);
}
}
-
+
private void parseLog(File log) throws Exception {
BufferedReader reader = new BufferedReader(new FileReader(log));
-
- String line;
- TreeMap<Long,Long> tm = null;
-
- while ((line = reader.readLine()) != null) {
- if (!line.startsWith("UUID"))
- continue;
- String[] tokens = line.split("\\s");
- String time = tokens[1];
- String uuid = tokens[2];
-
- if (flushes.containsKey(uuid)) {
- System.err.println("WARN Duplicate uuid " + log);
+ try {
+ String line;
+ TreeMap<Long,Long> tm = null;
+
+ while ((line = reader.readLine()) != null) {
+ if (!line.startsWith("UUID"))
+ continue;
+ String[] tokens = line.split("\\s");
+ String time = tokens[1];
+ String uuid = tokens[2];
+
+ if (flushes.containsKey(uuid)) {
+ System.err.println("WARN Duplicate uuid " + log);
+ return;
+ }
+
+ tm = new TreeMap<Long,Long>(Collections.reverseOrder());
+ tm.put(0l, Long.parseLong(time));
+ flushes.put(uuid, tm);
+ break;
+
+ }
+
+ if (tm == null) {
+ System.err.println("WARN Bad ingest log " + log);
return;
}
-
- tm = new TreeMap<Long,Long>(Collections.reverseOrder());
- tm.put(0l, Long.parseLong(time));
- flushes.put(uuid, tm);
- break;
-
- }
-
- if (tm == null) {
- System.err.println("WARN Bad ingest log " + log);
- return;
- }
-
- while ((line = reader.readLine()) != null) {
- String[] tokens = line.split("\\s");
-
- if (!tokens[0].equals("FLUSH"))
- continue;
-
- String time = tokens[1];
- String count = tokens[4];
-
- tm.put(Long.parseLong(count), Long.parseLong(time));
+
+ while ((line = reader.readLine()) != null) {
+ String[] tokens = line.split("\\s");
+
+ if (!tokens[0].equals("FLUSH"))
+ continue;
+
+ String time = tokens[1];
+ String count = tokens[4];
+
+ tm.put(Long.parseLong(count), Long.parseLong(time));
+ }
+ } finally {
+ if (reader != null)
+ reader.close();
}
-
}
-
+
Iterator<Long> getTimes(String uuid, long count) {
TreeMap<Long,Long> tm = flushes.get(uuid);
-
+
if (tm == null)
return null;
-
+
return tm.tailMap(count).values().iterator();
}
}
-
+
static class TabletAssignment {
String tablet;
String endRow;
String prevEndRow;
String server;
long time;
-
+
TabletAssignment(String tablet, String er, String per, String server, long time) {
this.tablet = new String(tablet);
this.endRow = new String(er);
@@ -146,133 +150,139 @@ public class UndefinedAnalyzer {
this.server = new String(server);
this.time = time;
}
-
+
public boolean contains(String row) {
return prevEndRow.compareTo(row) < 0 && endRow.compareTo(row) >= 0;
}
}
-
+
static class TabletHistory {
-
+
List<TabletAssignment> assignments = new ArrayList<TabletAssignment>();
-
+
TabletHistory(String tableId, String acuLogDir) throws Exception {
File dir = new File(acuLogDir);
File[] masterLogs = dir.listFiles(new FilenameFilter() {
+ @Override
public boolean accept(java.io.File dir, String name) {
return name.matches("master.*debug.log.*");
}
});
-
+
SimpleDateFormat sdf = new SimpleDateFormat("dd HH:mm:ss,SSS yyyy MM");
String currentYear = (Calendar.getInstance().get(Calendar.YEAR)) + "";
String currentMonth = (Calendar.getInstance().get(Calendar.MONTH) + 1) + "";
-
+
for (File masterLog : masterLogs) {
-
+
BufferedReader reader = new BufferedReader(new FileReader(masterLog));
- String line;
- while ((line = reader.readLine()) != null) {
- if (line.contains("TABLET_LOADED")) {
- String[] tokens = line.split("\\s+");
- String tablet = tokens[8];
- String server = tokens[10];
-
- int pos1 = -1;
- int pos2 = -1;
- int pos3 = -1;
-
- for (int i = 0; i < tablet.length(); i++) {
- if (tablet.charAt(i) == '<' || tablet.charAt(i) == ';') {
- if (pos1 == -1) {
- pos1 = i;
- } else if (pos2 == -1) {
- pos2 = i;
- } else {
- pos3 = i;
+ try {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ if (line.contains("TABLET_LOADED")) {
+ String[] tokens = line.split("\\s+");
+ String tablet = tokens[8];
+ String server = tokens[10];
+
+ int pos1 = -1;
+ int pos2 = -1;
+ int pos3 = -1;
+
+ for (int i = 0; i < tablet.length(); i++) {
+ if (tablet.charAt(i) == '<' || tablet.charAt(i) == ';') {
+ if (pos1 == -1) {
+ pos1 = i;
+ } else if (pos2 == -1) {
+ pos2 = i;
+ } else {
+ pos3 = i;
+ }
}
}
- }
-
- if (pos1 > 0 && pos2 > 0 && pos3 == -1) {
- String tid = tablet.substring(0, pos1);
- String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2);
- String prevEndRow = tablet.charAt(pos2) == '<' ? "" : tablet.substring(pos2 + 1);
- if (tid.equals(tableId)) {
- // System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow);
- Date date = sdf.parse(tokens[0] + " " + tokens[1] + " " + currentYear + " " + currentMonth);
- // System.out.println(" "+date);
-
- assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime()));
-
+
+ if (pos1 > 0 && pos2 > 0 && pos3 == -1) {
+ String tid = tablet.substring(0, pos1);
+ String endRow = tablet.charAt(pos1) == '<' ? "8000000000000000" : tablet.substring(pos1 + 1, pos2);
+ String prevEndRow = tablet.charAt(pos2) == '<' ? "" : tablet.substring(pos2 + 1);
+ if (tid.equals(tableId)) {
+ // System.out.println(" "+server+" "+tid+" "+endRow+" "+prevEndRow);
+ Date date = sdf.parse(tokens[0] + " " + tokens[1] + " " + currentYear + " " + currentMonth);
+ // System.out.println(" "+date);
+
+ assignments.add(new TabletAssignment(tablet, endRow, prevEndRow, server, date.getTime()));
+
+ }
+ } else if (!tablet.startsWith("!0")) {
+ System.err.println("Cannot parse tablet " + tablet);
}
- } else if (!tablet.startsWith("!0")) {
- System.err.println("Cannot parse tablet " + tablet);
+
}
-
}
+ } finally {
+ if (reader != null)
+ reader.close();
}
}
}
-
+
TabletAssignment findMostRecentAssignment(String row, long time1, long time2) {
-
+
long latest = Long.MIN_VALUE;
TabletAssignment ret = null;
-
+
for (TabletAssignment assignment : assignments) {
if (assignment.contains(row) && assignment.time <= time2 && assignment.time > latest) {
latest = assignment.time;
ret = assignment;
}
}
-
+
return ret;
}
}
-
+
public static void main(String[] args) throws Exception {
-
+
if (args.length != 7) {
System.err.println("Usage : " + UndefinedAnalyzer.class.getName() + " <instance> <zoo> <user> <pass> <table> <ci log dir> <acu log dir>");
return;
}
-
+
String instanceName = args[0];
String zooKeepers = args[1];
-
+
String user = args[2];
String password = args[3];
-
+
String table = args[4];
String logDir = args[5];
String acuLogDir = args[6];
-
+
List<UndefinedNode> undefs = new ArrayList<UndefinedNode>();
-
+
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
String line;
while ((line = reader.readLine()) != null) {
String[] tokens = line.split("\\s");
String undef = tokens[0];
String ref = tokens[1];
-
+
undefs.add(new UndefinedNode(undef, ref));
}
-
+
ZooKeeperInstance zki = new ZooKeeperInstance(instanceName, zooKeepers);
Connector conn = zki.getConnector(user, password.getBytes());
BatchScanner bscanner = conn.createBatchScanner(table, Constants.NO_AUTHS, 20);
-
+
List<Range> refs = new ArrayList<Range>();
-
+
for (UndefinedNode undefinedNode : undefs)
refs.add(new Range(new Text(undefinedNode.ref)));
-
+
bscanner.setRanges(refs);
-
+
HashMap<String,List<String>> refInfo = new HashMap<String,List<String>>();
-
+
for (Entry<Key,Value> entry : bscanner) {
String ref = entry.getKey().getRow().toString();
List<String> vals = refInfo.get(ref);
@@ -280,32 +290,32 @@ public class UndefinedAnalyzer {
vals = new ArrayList<String>();
refInfo.put(ref, vals);
}
-
+
vals.add(entry.getValue().toString());
}
-
+
bscanner.close();
-
+
IngestInfo ingestInfo = new IngestInfo(logDir);
TabletHistory tabletHistory = new TabletHistory(Tables.getTableId(zki, table), acuLogDir);
-
+
SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
-
+
for (UndefinedNode undefinedNode : undefs) {
-
+
List<String> refVals = refInfo.get(undefinedNode.ref);
if (refVals != null) {
for (String refVal : refVals) {
TabletAssignment ta = null;
-
+
String[] tokens = refVal.split(":");
-
+
String uuid = tokens[0];
String count = tokens[1];
-
+
String t1 = "";
String t2 = "";
-
+
Iterator<Long> times = ingestInfo.getTimes(uuid, Long.parseLong(count, 16));
if (times != null) {
if (times.hasNext()) {
@@ -318,19 +328,19 @@ public class UndefinedAnalyzer {
}
}
}
-
+
if (ta == null)
System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + uuid + " " + t1 + " " + t2);
else
System.out.println(undefinedNode.undef + " " + undefinedNode.ref + " " + ta.tablet + " " + ta.server + " " + uuid + " " + t1 + " " + t2);
-
+
}
} else {
System.out.println(undefinedNode.undef + " " + undefinedNode.ref);
}
-
+
}
-
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/335f693a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java b/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
index 3c3d8d7..4a0f5ec 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/util/SendLogToChainsaw.java
@@ -18,8 +18,10 @@ package org.apache.accumulo.server.util;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FilenameFilter;
+import java.io.IOException;
import java.net.Socket;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
@@ -48,33 +50,33 @@ import org.apache.log4j.varia.LevelRangeFilter;
import org.apache.log4j.xml.XMLLayout;
public class SendLogToChainsaw extends XMLLayout {
-
+
private static Pattern logPattern = Pattern.compile(
"^(\\d\\d)\\s(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\s\\[(.*)\\]\\s(TRACE|DEBUG|INFO|WARN|FATAL|ERROR)\\s*?:(.*)$", Pattern.UNIX_LINES);
-
+
private File[] logFiles = null;
-
+
private SocketFactory factory = SocketFactory.getDefault();
-
+
private WildcardFileFilter fileFilter = null;
-
+
private Socket socket = null;
-
+
private Pattern lineFilter = null;
-
+
private LongRange dateFilter = null;
-
+
private LevelRangeFilter levelFilter = null;
-
+
public SendLogToChainsaw(String directory, String fileNameFilter, String host, int port, Date start, Date end, String regex, String level) throws Exception {
-
+
// Set up the file name filter
if (null != fileNameFilter) {
fileFilter = new WildcardFileFilter(fileNameFilter);
} else {
fileFilter = new WildcardFileFilter("*");
}
-
+
// Get the list of files that match
File dir = new File(directory);
if (dir.isDirectory()) {
@@ -82,22 +84,22 @@ public class SendLogToChainsaw extends XMLLayout {
} else {
throw new IllegalArgumentException(directory + " is not a directory or is not readable.");
}
-
+
if (logFiles.length == 0) {
throw new IllegalArgumentException("No files match the supplied filter.");
}
-
+
socket = factory.createSocket(host, port);
-
+
lineFilter = Pattern.compile(regex);
-
+
// Create Date Filter
if (null != start) {
if (end == null)
end = new Date(System.currentTimeMillis());
dateFilter = new LongRange(start.getTime(), end.getTime());
}
-
+
if (null != level) {
Level base = Level.toLevel(level.toUpperCase());
levelFilter = new LevelRangeFilter();
@@ -106,43 +108,60 @@ public class SendLogToChainsaw extends XMLLayout {
levelFilter.setLevelMax(Level.FATAL);
}
}
-
+
public void processLogFiles() throws Exception {
- for (File log : logFiles) {
- // Parse the server type and name from the log file name
- String threadName = log.getName().substring(0, log.getName().indexOf("."));
- FileReader fReader = new FileReader(log);
- BufferedReader reader = new BufferedReader(fReader);
-
- String line = reader.readLine();
- while (null != line) {
- String out = null;
+ String line = null;
+ String out = null;
+ FileReader fReader = null;
+ BufferedReader reader = null;
+ try {
+ for (File log : logFiles) {
+ // Parse the server type and name from the log file name
+ String threadName = log.getName().substring(0, log.getName().indexOf("."));
+ try {
+ fReader = new FileReader(log);
+ } catch (FileNotFoundException e) {
+ System.out.println("Unable to find file: " + log.getAbsolutePath());
+ throw e;
+ }
+ reader = new BufferedReader(fReader);
+
try {
- out = convertLine(line, threadName);
- if (null != out) {
- if (socket != null && socket.isConnected())
- socket.getOutputStream().write(out.getBytes());
- else
- System.err.println("Unable to send data to transport");
+ line = reader.readLine();
+ while (null != line) {
+ out = convertLine(line, threadName);
+ if (null != out) {
+ if (socket != null && socket.isConnected())
+ socket.getOutputStream().write(out.getBytes());
+ else
+ System.err.println("Unable to send data to transport");
+ }
+ line = reader.readLine();
}
- } catch (Exception e) {
+ } catch (IOException e) {
System.out.println("Error processing line: " + line + ". Output was " + out);
throw e;
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ if (fReader != null) {
+ fReader.close();
+ }
}
- line = reader.readLine();
}
- reader.close();
- fReader.close();
+ } finally {
+ if (socket != null && socket.isConnected()) {
+ socket.close();
+ }
}
- if (socket != null && socket.isConnected())
- socket.close();
}
-
+
private String convertLine(String line, String threadName) throws Exception {
String result = null;
Matcher m = logPattern.matcher(line);
if (m.matches()) {
-
+
Calendar cal = Calendar.getInstance();
cal.setTime(new Date(System.currentTimeMillis()));
Integer date = Integer.parseInt(m.group(1));
@@ -188,54 +207,54 @@ public class SendLogToChainsaw extends XMLLayout {
}
return result;
}
-
+
private static Options getOptions() {
Options opts = new Options();
-
+
Option dirOption = new Option("d", "logDirectory", true, "ACCUMULO log directory path");
dirOption.setArgName("dir");
dirOption.setRequired(true);
opts.addOption(dirOption);
-
+
Option fileFilterOption = new Option("f", "fileFilter", true, "filter to apply to names of logs");
fileFilterOption.setArgName("filter");
fileFilterOption.setRequired(false);
opts.addOption(fileFilterOption);
-
+
Option hostOption = new Option("h", "host", true, "host where chainsaw is running");
hostOption.setArgName("hostname");
hostOption.setRequired(true);
opts.addOption(hostOption);
-
+
Option portOption = new Option("p", "port", true, "port where XMLSocketReceiver is listening");
portOption.setArgName("portnum");
portOption.setRequired(true);
opts.addOption(portOption);
-
+
Option startOption = new Option("s", "start", true, "start date filter (yyyyMMddHHmmss)");
startOption.setArgName("date");
startOption.setRequired(true);
opts.addOption(startOption);
-
+
Option endOption = new Option("e", "end", true, "end date filter (yyyyMMddHHmmss)");
endOption.setArgName("date");
endOption.setRequired(true);
opts.addOption(endOption);
-
+
Option levelOption = new Option("l", "level", true, "filter log level");
levelOption.setArgName("level");
levelOption.setRequired(false);
opts.addOption(levelOption);
-
+
Option msgFilter = new Option("m", "messageFilter", true, "regex filter for log messages");
msgFilter.setArgName("regex");
msgFilter.setRequired(false);
opts.addOption(msgFilter);
-
+
return opts;
-
+
}
-
+
/**
*
* @param args
@@ -245,11 +264,11 @@ public class SendLogToChainsaw extends XMLLayout {
* @throws Exception
*/
public static void main(String[] args) throws Exception {
-
+
Options o = getOptions();
CommandLine cl = null;
cl = new BasicParser().parse(o, args);
-
+
String logDir = cl.getOptionValue(o.getOption("d").getOpt());
String fileNameFilter = null;
if (cl.hasOption(o.getOption("f").getOpt()))
@@ -277,9 +296,9 @@ public class SendLogToChainsaw extends XMLLayout {
String levelFilter = null;
if (cl.hasOption(o.getOption("l").getOpt()))
levelFilter = cl.getOptionValue(o.getOption("l").getOpt());
-
+
SendLogToChainsaw c = new SendLogToChainsaw(logDir, fileNameFilter, chainsawHost, chainsawPort, startDate, endDate, msgFilter, levelFilter);
c.processLogFiles();
}
-
+
}