You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by ph...@apache.org on 2010/06/09 17:26:25 UTC
svn commit: r953041 [1/6] - in /hadoop/zookeeper/trunk: ./
src/contrib/loggraph/ src/contrib/loggraph/bin/ src/contrib/loggraph/src/
src/contrib/loggraph/src/java/ src/contrib/loggraph/src/java/org/
src/contrib/loggraph/src/java/org/apache/ src/contrib...
Author: phunt
Date: Wed Jun 9 15:26:22 2010
New Revision: 953041
URL: http://svn.apache.org/viewvc?rev=953041&view=rev
Log:
ZOOKEEPER-773. Log visualisation
Added:
hadoop/zookeeper/trunk/src/contrib/loggraph/
hadoop/zookeeper/trunk/src/contrib/loggraph/README.txt
hadoop/zookeeper/trunk/src/contrib/loggraph/bin/
hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh (with props)
hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh (with props)
hadoop/zookeeper/trunk/src/contrib/loggraph/build.xml
hadoop/zookeeper/trunk/src/contrib/loggraph/ivy.xml
hadoop/zookeeper/trunk/src/contrib/loggraph/src/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterException.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterParser.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/JsonGenerator.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JEntry.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JSource.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogEntry.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogIterator.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogServer.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSkipList.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSource.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MeasureThroughput.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MergedLogSource.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/RandomAccessFileReader.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TransactionEntry.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TxnLogSource.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/AndOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/Arg.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/EqualsOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/GreaterThanOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/LessThanOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/NotOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/NumberArg.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/OrOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/StringArg.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/SymbolArg.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/filterops/XorOp.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/FileLoader.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/Fs.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/GraphData.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/JsonServlet.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/NumEvents.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/StaticContent.java
hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/servlets/Throughput.java
hadoop/zookeeper/trunk/src/contrib/loggraph/web/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/log4j.properties
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/date.format.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/g.bar.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/g.dot.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/g.line.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/g.pie.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/g.raphael.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/load-big.gif (with props)
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/load.gif (with props)
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.css
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.log.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.server.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.session.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.stats.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/loggraph.ui.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/main.html
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/raphael.js
hadoop/zookeeper/trunk/src/contrib/loggraph/web/org/apache/zookeeper/graph/resources/yui-min.js
Modified:
hadoop/zookeeper/trunk/CHANGES.txt
Modified: hadoop/zookeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/CHANGES.txt?rev=953041&r1=953040&r2=953041&view=diff
==============================================================================
--- hadoop/zookeeper/trunk/CHANGES.txt (original)
+++ hadoop/zookeeper/trunk/CHANGES.txt Wed Jun 9 15:26:22 2010
@@ -80,6 +80,8 @@ NEW FEATURES:
ZOOKEEPER-464. Need procedure to garbage collect ledgers
(erwin via fpj)
+ ZOOKEEPER-773. Log visualisation (Ivan Kelly via phunt)
+
Release 3.3.0 - 2010-03-24
Non-backward compatible changes:
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/README.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/README.txt?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/README.txt (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/README.txt Wed Jun 9 15:26:22 2010
@@ -0,0 +1,69 @@
+LogGraph README
+
+1 - About
+LogGraph is an application for viewing and filtering zookeeper logs. It can handle transaction logs and message logs.
+
+2 - Compiling
+
+Run "ant jar" in src/contrib/loggraph/. This will download all dependencies and compile all the loggraph code.
+
+Once compilation has finished, you can run it the the loggraph.sh script in src/contrib/loggraph/bin. This will start and embedded web server on your machine.
+Navigate to http://localhost:8182/graph/main.html
+
+3 - Usage
+LogGraph presents the user with 4 views,
+
+ a) Simple log view
+ This view simply displays the log text. This isn't very useful without filters (see "Filtering the logs").
+
+ b) Server view
+ The server view shows the interactions between the different servers in an ensemble. The X axis represents time.
+ * Exceptions show up as red dots. Hovering your mouse over them will give you more details of the exception
+ * The colour of the line represents the election state of the server.
+ - orange means LOOKING for leader
+ - dark green means the server is the leader
+ - light green means the server is following a leader
+ - yellow means there isn't enough information to determine the state of the server.
+ * The gray arrows denote election messages between servers. Pink dashed arrows are messages that were sent but never delivered.
+
+ c) Session view
+ The session view shows the lifetime of sessions on a server. Use the time filter to narrow down the view. Any more than about 2000 events will take a long time to view in your browser.
+ The X axis represents time. Each line is a session. The black dots represent events on the session. You can click on the black dots for more details of the event.
+
+ d) Stats view
+ There is currently only one statistics view, Transactions/minute. Suggestions for other statistic views are very welcome.
+
+4 - Filtering the logs
+The logs can be filtered in 2 ways, by time and by content.
+
+To filter by time simply move the slider to the desired start time. The time window specifies how many milliseconds after and including the start time will be displayed.
+
+Content filtering uses a adhoc filtering language, using prefix notation. The language looks somewhat similar to lisp. A statement in the language takes the form (op arg arg ....). A statement resolves to a boolean value. Statements can be nested.
+
+4.1 - Filter arguments
+An argument can be a number, a string or a symbol. A number is any argument which starts with -, + or 0 to 9. If the number starts with 0x it is interpretted as hexidecimal. Otherwise it is interpretted as decimal. If the argument begins with a double-quote, (") it is interpretted as a string. Anything else is interpretted as a symbol.
+
+4.2 - Filter symbols
+The possible filter symbols are:
+
+client-id : number, the session id of the client who initiated a transaction.
+cxid : number, the cxid of a transaction
+zxid : number, the zxid of a transaction
+operation : string, the operation being performed, for example "setData", "createSession", "closeSession", "error", "create"
+
+4.3 - Filter operations
+The possible filter operations are:
+
+or : logical or, takes 1 or more arguments which must be other statements.
+and : logical and, takes 1 or more arguments which must be other statements.
+not : logical not, takes 1 argument which must be another statement.
+xor : exclusive or, takes 1 or more arguments which must be other statements.
+= : equals, takes 1 or more arguments, which must all be equal to each other to return true.
+> : greater than, takes 1 or more arguments, to return true the 1st argument must be greater than the 2nd argument which must be greater than the 3rd argument and so on...
+< : less than, takes 1 or more arguments, to return true the 1st argument must be less than the 2nd argument which must be less than the 3rd argument and so on...
+
+4.3 - Filter examples
+Give me all the setData operations with session id 0xdeadbeef or 0xcafeb33r but not with zxid 0x12341234 ->
+
+(and (= operation "setData") (or (= client-id 0xdeadbeef) (= client-id 0xcafeb33r)) (not (= zxid 0x12341234)))
+
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh Wed Jun 9 15:26:22 2010
@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+make_canonical () {
+ cd $1; pwd;
+}
+
+SCRIPTDIR=`dirname $0`
+BUILDDIR=`make_canonical $SCRIPTDIR/../../../../build/contrib/loggraph`
+LIBDIR=`make_canonical $BUILDDIR/lib`
+WEBDIR=`make_canonical $SCRIPTDIR/../web`
+ZKDIR=`make_canonical $SCRIPTDIR/../../../../build/`
+
+if [ ! -x $BUILDDIR ]; then
+ echo "\n\n*** You need to build loggraph before running it ***\n\n";
+ exit;
+fi
+
+for i in `ls $LIBDIR`; do
+ CLASSPATH=$LIBDIR/$i:$CLASSPATH
+done
+
+for i in $ZKDIR/zookeeper-*.jar; do
+ CLASSPATH="$i:$CLASSPATH"
+done
+
+CLASSPATH=$BUILDDIR/classes:$WEBDIR:$CLASSPATH
+echo $CLASSPATH
+java -Dlog4j.configuration=org/apache/zookeeper/graph/log4j.properties -Xdebug -Xrunjdwp:transport=dt_socket,address=4444,server=y,suspend=n -cp $CLASSPATH org.apache.zookeeper.graph.LogServer $*
Propchange: hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph-dev.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh Wed Jun 9 15:26:22 2010
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+make_canonical () {
+ cd $1; pwd;
+}
+
+SCRIPTDIR=`dirname $0`
+BUILDDIR=`make_canonical $SCRIPTDIR/../../../../build/contrib/loggraph`
+LIBDIR=`make_canonical $BUILDDIR/lib`
+ZKDIR=`make_canonical $SCRIPTDIR/../../../../build/`
+
+if [ ! -x $BUILDDIR ]; then
+ echo "\n\n*** You need to build loggraph before running it ***\n\n";
+ exit;
+fi
+
+for i in `ls $LIBDIR`; do
+ CLASSPATH=$LIBDIR/$i:$CLASSPATH
+done
+
+for i in `ls $BUILDDIR/*.jar`; do
+ CLASSPATH=$i:$CLASSPATH
+done
+
+for i in $ZKDIR/zookeeper-*.jar; do
+ CLASSPATH="$i:$CLASSPATH"
+done
+
+java -cp $CLASSPATH org.apache.zookeeper.graph.LogServer $*
+
+
+
+
Propchange: hadoop/zookeeper/trunk/src/contrib/loggraph/bin/loggraph.sh
------------------------------------------------------------------------------
svn:executable = *
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/build.xml
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/build.xml?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/build.xml (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/build.xml Wed Jun 9 15:26:22 2010
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="loggraph" default="jar">
+
+ <import file="../build-contrib.xml"/>
+
+ <target name="init" depends="check-contrib,zookeeperbuildcontrib.init" unless="skip.contrib">
+ <echo message="contrib: ${name}"/>
+ <mkdir dir="${build.dir}"/>
+ <antcall target="init-contrib"/>
+ </target>
+
+ <target name="compile" depends="init,ivy-retrieve,zookeeperbuildcontrib.compile" unless="skip.contrib">
+ </target>
+
+ <target name="setjarname">
+ <property name="jarname" value="${build.dir}/zookeeper-${version}-${name}.jar"/>
+ </target>
+
+ <target name="jar" depends="setjarname,compile" >
+ <jar destfile="${jarname}">
+ <fileset file="${zk.root}/LICENSE.txt" />
+ <fileset dir="${build.classes}" />
+ <fileset dir="web"/>
+ <manifest>
+ <attribute name="Built-By" value="${user.name}"/>
+ <attribute name="Built-At" value="${build.time}"/>
+ <attribute name="Built-On" value="${host.name}" />
+ <attribute name="Implementation-Title" value="org.apache.zookeeper.graph"/>
+ <attribute name="Implementation-Version" value="${revision}"/>
+ <attribute name="Implementation-Vendor" value="The Apache Software Foundation"/>
+ </manifest>
+ </jar>
+ </target>
+
+ <target name="test">
+ <echo message="No test target defined for this package" />
+ </target>
+
+
+ <target name="package" depends="compile, zookeeperbuildcontrib.package" unless="skip.contrib">
+ <echo message="contrib: ${name}"/>
+
+ <copy file="${basedir}/build.xml" todir="${dist.dir}/contrib/${name}"/>
+
+ <mkdir dir="${dist.dir}/contrib/${name}/src"/>
+ <copy todir="${dist.dir}/contrib/${name}/src">
+ <fileset dir="${basedir}/src"/>
+ </copy>
+
+ </target>
+
+</project>
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/ivy.xml?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/ivy.xml (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/ivy.xml Wed Jun 9 15:26:22 2010
@@ -0,0 +1,41 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<ivy-module version="2.0"
+ xmlns:e="http://ant.apache.org/ivy/extra">
+
+ <info organisation="org.apache.zookeeper"
+ module="${name}" revision="${version}">
+ <license name="Apache 2.0"/>
+ <ivyauthor name="Apache Hadoop" url="http://hadoop.apache.org"/>
+ <description>ZooKeeper Graphing</description>
+ </info>
+
+ <configurations defaultconfmapping="default">
+ <conf name="default"/>
+ <conf name="test"/>
+ </configurations>
+
+ <dependencies>
+ <!-- transitive false turns off dependency checking, log4j deps seem borked -->
+ <dependency org="log4j" name="log4j" rev="1.2.15" transitive="false"/>
+ <dependency org="org.eclipse.jetty" name="jetty-server" rev="7.0.1.v20091125" />
+ <dependency org="org.eclipse.jetty" name="jetty-servlet" rev="7.0.1.v20091125" />
+ <dependency org="com.googlecode.json-simple" name="json-simple" rev="1.1" />
+ </dependencies>
+
+</ivy-module>
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterException.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterException.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterException.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterException.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+public class FilterException extends Exception {
+ public FilterException(String s) { super(s); }
+};
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterOp.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterOp.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterOp.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterOp.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.util.ArrayList;
+import org.apache.zookeeper.graph.filterops.*;
+
+public abstract class FilterOp {
+ protected ArrayList<FilterOp> subOps;
+ protected ArrayList<Arg> args;
+
+ public enum ArgType {
+ STRING, NUMBER, SYMBOL
+ }
+
+ public FilterOp() {
+ subOps = new ArrayList<FilterOp>();
+ args = new ArrayList<Arg>();
+ }
+
+ public static FilterOp newOp(String op) throws FilterException {
+ if (op.equals("or"))
+ return new OrOp();
+ if (op.equals("and"))
+ return new AndOp();
+ if (op.equals("not"))
+ return new NotOp();
+ if (op.equals("xor"))
+ return new XorOp();
+ if (op.equals("="))
+ return new EqualsOp();
+ if (op.equals("<"))
+ return new LessThanOp();
+ if (op.equals(">"))
+ return new GreaterThanOp();
+
+ throw new FilterException("Invalid operation '"+op+"'");
+ }
+
+ public void addSubOp(FilterOp op) {
+ subOps.add(op);
+ }
+
+ public void addArg(Arg arg) {
+ args.add(arg);
+ }
+
+ public abstract boolean matches(LogEntry entry) throws FilterException;
+
+ public String toString() {
+ String op = "(" + getClass().getName();
+ for (FilterOp f : subOps) {
+ op += " " + f;
+ }
+ for (Arg a : args) {
+ op += " " + a;
+ }
+ return op + ")";
+ }
+}
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterParser.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterParser.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterParser.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/FilterParser.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.PushbackReader;
+import java.io.StringReader;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.zookeeper.graph.filterops.*;
+
+public class FilterParser {
+ private PushbackReader reader;
+
+ public FilterParser(String s) {
+ reader = new PushbackReader(new StringReader(s));
+ }
+
+ private String readUntilSpace() throws IOException {
+ StringBuffer buffer = new StringBuffer();
+
+ int c = reader.read();
+ while (!Character.isWhitespace(c) && c != ')' && c != '(') {
+ buffer.append((char)c);
+ c = reader.read();
+ if (c == -1) {
+ break;
+ }
+ }
+ reader.unread(c);
+
+ return buffer.toString().trim();
+ }
+
+ private StringArg readStringArg() throws IOException, FilterException {
+ int c = reader.read();
+ int last = 0;
+ if (c != '"') {
+ throw new FilterException("Check the parser, trying to read a string that doesn't begin with quotes");
+ }
+ StringBuffer buffer = new StringBuffer();
+ while (reader.ready()) {
+ last = c;
+ c = reader.read();
+ if (c == -1) {
+ break;
+ }
+
+ if (c == '"' && last != '\\') {
+ return new StringArg(buffer.toString());
+ } else {
+ buffer.append((char)c);
+ }
+ }
+ throw new FilterException("Unterminated string");
+ }
+
+ private NumberArg readNumberArg() throws IOException, FilterException {
+ String strval = readUntilSpace();
+
+ try {
+ if (strval.startsWith("0x")) {
+ return new NumberArg(Long.valueOf(strval.substring(2), 16));
+ } else {
+ return new NumberArg(Long.valueOf(strval));
+ }
+ } catch (NumberFormatException e) {
+ throw new FilterException("Not a number [" + strval + "]\n" + e);
+ }
+ }
+
+ private SymbolArg readSymbolArg() throws IOException, FilterException {
+ return new SymbolArg(readUntilSpace());
+ }
+
+ public FilterOp parse() throws IOException, FilterException {
+ int c = reader.read();
+ if (c != '(') {
+ throw new FilterException("Invalid format");
+ }
+
+ String opstr = readUntilSpace();
+ FilterOp op = FilterOp.newOp(opstr);
+
+ while (reader.ready()) {
+ c = reader.read();
+ if (c == -1) {
+ break;
+ }
+ if (c == '(') {
+ reader.unread(c);
+ op.addSubOp(parse());
+ } else if (c == ')') {
+ return op;
+ } else if (c == '"') {
+ reader.unread(c);
+ op.addArg(readStringArg());
+ } else if (Character.isDigit(c) || c == '-' || c == '+') {
+ reader.unread(c);
+ op.addArg(readNumberArg());
+ } else if (Character.isJavaIdentifierStart(c)) {
+ reader.unread(c);
+ op.addArg(readSymbolArg());
+ }
+ }
+ throw new FilterException("Incomplete filter");
+ }
+
+ public static void main(String[] args) throws IOException, FilterException {
+ if (args.length == 1) {
+ System.out.println(new FilterParser(args[0]).parse());
+ } else {
+ System.out.println(new FilterParser("(or (and (= session foobar) (= session barfoo)) (= session sdfs))").parse());
+ }
+ }
+};
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/JsonGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/JsonGenerator.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/JsonGenerator.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/JsonGenerator.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+import java.io.Writer;
+import java.io.OutputStreamWriter;
+import java.io.IOException;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.ListIterator;
+
+public class JsonGenerator {
+ private JSONObject root;
+ private HashSet<Integer> servers;
+
+ private class Message {
+ private int from;
+ private int to;
+ private long zxid;
+
+ public Message(int from, int to, long zxid) {
+ this.from = from;
+ this.to = to;
+ this.zxid = zxid;
+ }
+
+ public boolean equals(Message m) {
+ return (m.from == this.from
+ && m.to == this.to
+ && m.zxid == this.zxid);
+ }
+ };
+
+ public JSONObject txnEntry(TransactionEntry e) {
+ JSONObject event = new JSONObject();
+
+ event.put("time", Long.toString(e.getTimestamp()));
+ event.put("client", Long.toHexString(e.getClientId()));
+ event.put("cxid", Long.toHexString(e.getCxid()));
+ event.put("zxid", Long.toHexString(e.getZxid()));
+ event.put("op", e.getOp());
+ event.put("extra", e.getExtra());
+ event.put("type", "transaction");
+
+ return event;
+ }
+
+ /**
+ Assumes entries are sorted by timestamp.
+ */
+ public JsonGenerator(LogIterator iter) {
+ servers = new HashSet<Integer>();
+
+ Pattern stateChangeP = Pattern.compile("- (LOOKING|FOLLOWING|LEADING)");
+ Pattern newElectionP = Pattern.compile("New election. My id = (\\d+), Proposed zxid = (\\d+)");
+ Pattern receivedProposalP = Pattern.compile("Notification: \\d+, (\\d+), (\\d+), \\d+, [^,]*, [^,]*, (\\d+)");//, LOOKING, LOOKING, 2
+ Pattern exceptionP = Pattern.compile("xception");
+
+ root = new JSONObject();
+ Matcher m = null;
+ JSONArray events = new JSONArray();
+ root.put("events", events);
+
+ long starttime = Long.MAX_VALUE;
+ long endtime = 0;
+
+ int leader = 0;
+ long curEpoch = 0;
+ boolean newEpoch = false;
+
+ while (iter.hasNext()) {
+ LogEntry ent = iter.next();
+
+ if (ent.getTimestamp() < starttime) {
+ starttime = ent.getTimestamp();
+ }
+ if (ent.getTimestamp() > endtime) {
+ endtime = ent.getTimestamp();
+ }
+
+ if (ent.getType() == LogEntry.Type.TXN) {
+ events.add(txnEntry((TransactionEntry)ent));
+ } else {
+ Log4JEntry e = (Log4JEntry)ent;
+ servers.add(e.getNode());
+
+ if ((m = stateChangeP.matcher(e.getEntry())).find()) {
+ JSONObject stateChange = new JSONObject();
+ stateChange.put("type", "stateChange");
+ stateChange.put("time", e.getTimestamp());
+ stateChange.put("server", e.getNode());
+ stateChange.put("state", m.group(1));
+ events.add(stateChange);
+
+ if (m.group(1).equals("LEADING")) {
+ leader = e.getNode();
+ }
+ } else if ((m = newElectionP.matcher(e.getEntry())).find()) {
+ Iterator<Integer> iterator = servers.iterator();
+ long zxid = Long.valueOf(m.group(2));
+ int count = (int)zxid;// & 0xFFFFFFFFL;
+ int epoch = (int)Long.rotateRight(zxid, 32);// >> 32;
+
+ if (leader != 0 && epoch > curEpoch) {
+ JSONObject stateChange = new JSONObject();
+ stateChange.put("type", "stateChange");
+ stateChange.put("time", e.getTimestamp());
+ stateChange.put("server", leader);
+ stateChange.put("state", "INIT");
+ events.add(stateChange);
+ leader = 0;
+ }
+
+ while (iterator.hasNext()) {
+ int dst = iterator.next();
+ if (dst != e.getNode()) {
+ JSONObject msg = new JSONObject();
+ msg.put("type", "postmessage");
+ msg.put("src", e.getNode());
+ msg.put("dst", dst);
+ msg.put("time", e.getTimestamp());
+ msg.put("zxid", m.group(2));
+ msg.put("count", count);
+ msg.put("epoch", epoch);
+
+ events.add(msg);
+ }
+ }
+ } else if ((m = receivedProposalP.matcher(e.getEntry())).find()) {
+ // Pattern.compile("Notification: \\d+, (\\d+), (\\d+), \\d+, [^,]*, [^,]*, (\\d+)");//, LOOKING, LOOKING, 2
+ int src = Integer.valueOf(m.group(3));
+ long zxid = Long.valueOf(m.group(1));
+ int dst = e.getNode();
+ long epoch2 = Long.valueOf(m.group(2));
+
+ int count = (int)zxid;// & 0xFFFFFFFFL;
+ int epoch = (int)Long.rotateRight(zxid, 32);// >> 32;
+
+ if (leader != 0 && epoch > curEpoch) {
+ JSONObject stateChange = new JSONObject();
+ stateChange.put("type", "stateChange");
+ stateChange.put("time", e.getTimestamp());
+ stateChange.put("server", leader);
+ stateChange.put("state", "INIT");
+ events.add(stateChange);
+ leader = 0;
+ }
+
+ // System.out.println("src: "+src+" dst: "+dst+ " zxid: "+zxid);
+
+ // System.out.println(cur);
+ if (src != dst) {
+ JSONObject msg = new JSONObject();
+ msg.put("type", "delivermessage");
+ msg.put("src", src);
+ msg.put("dst", dst);
+ msg.put("time", e.getTimestamp());
+ msg.put("zxid", zxid);
+ msg.put("epoch", epoch);
+ msg.put("count", count);
+ msg.put("epoch2", epoch2);
+
+ events.add(msg);
+ }
+ } else if ((m = exceptionP.matcher(e.getEntry())).find()) {
+ JSONObject ex = new JSONObject();
+ ex.put("type", "exception");
+ ex.put("server", e.getNode());
+ ex.put("time", e.getTimestamp());
+ ex.put("text", e.getEntry());
+ events.add(ex);
+ }
+ }
+ JSONObject ex = new JSONObject();
+ ex.put("type", "text");
+ ex.put("time", ent.getTimestamp());
+ String txt = ent.toString();
+ ex.put("text", txt);
+ events.add(ex);
+ }
+ // System.out.println("pending messages: "+pendingMessages.size());
+ root.put("starttime", starttime);
+ root.put("endtime", endtime);
+
+ JSONArray serversarray = new JSONArray();
+ root.put("servers", serversarray);
+
+ Iterator<Integer> iterator = servers.iterator();
+ while (iterator.hasNext()) {
+ serversarray.add(iterator.next());
+ }
+ }
+
+ public String toString() {
+ return JSONValue.toJSONString(root);
+ }
+
+ public static void main(String[] args) throws Exception {
+ MergedLogSource src = new MergedLogSource(args);
+ LogIterator iter = src.iterator();
+ System.out.println(new JsonGenerator(iter));
+ }
+}
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JEntry.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JEntry.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JEntry.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JEntry.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+public class Log4JEntry extends LogEntry {
+ public Log4JEntry(long timestamp, int node, String entry) {
+ super(timestamp);
+ setAttribute("log-text", entry);
+ setAttribute("node", new Integer(node));
+ }
+
+ public String getEntry() {
+ return (String) getAttribute("log-text");
+ }
+
+ public String toString() {
+ return "" + getTimestamp() + "::::" + getNode() + "::::" + getEntry();
+ }
+
+ public int getNode() {
+ return (Integer) getAttribute("node");
+ }
+
+ public Type getType() { return LogEntry.Type.LOG4J; }
+}
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JSource.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JSource.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JSource.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/Log4JSource.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,380 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.File;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import java.util.ArrayList;
+import java.util.Date;
+import java.text.SimpleDateFormat;
+import java.text.ParseException;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+
+import java.io.EOFException;
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.log4j.Logger;
+
+public class Log4JSource implements LogSource {
+ private static final Logger LOG = Logger.getLogger(Log4JSource.class);
+
+ private static final int skipN = 10000;
+ private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
+
+ private LogSkipList skiplist = null;
+
+ private String file = null;
+ private long starttime = 0;
+ private long endtime = 0;
+ private int serverid = 0;
+ private long size = 0;
+
+ private Pattern timep;
+
+ public boolean overlapsRange(long starttime, long endtime) {
+ return (starttime <= this.endtime && endtime >= this.starttime);
+ }
+
+ public long size() { return size; }
+ public long getStartTime() { return starttime; }
+ public long getEndTime() { return endtime; }
+ public LogSkipList getSkipList() { return skiplist; }
+
+ private class Log4JSourceIterator implements LogIterator {
+ private RandomAccessFileReader in;
+ private LogEntry next = null;
+ private long starttime = 0;
+ private long endtime = 0;
+ private String buf = "";
+ private Log4JSource src = null;
+ private long skippedAtStart = 0;
+ private SimpleDateFormat dateformat = null;
+ private FilterOp filter = null;
+
+ public Log4JSourceIterator(Log4JSource src, long starttime, long endtime) throws IllegalArgumentException, FilterException {
+ this(src, starttime, endtime, null);
+ }
+
+ public Log4JSourceIterator(Log4JSource src, long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException {
+
+ this.dateformat = new SimpleDateFormat(DATE_FORMAT);
+ this.src = src;
+ this.starttime = starttime;
+ this.endtime = endtime;
+
+ File f = new File(src.file);
+ try {
+ in = new RandomAccessFileReader(f);
+ } catch (FileNotFoundException e) {
+ throw new IllegalArgumentException("Bad file passed in (" + src.file +") cannot open:" + e);
+ }
+
+ // skip to the offset of latest skip point before starttime
+ LogSkipList.Mark start = src.getSkipList().findMarkBefore(starttime);
+ try {
+ in.seek(start.getBytes());
+ skippedAtStart = start.getEntriesSkipped();
+ } catch (IOException ioe) {
+ // if we can't skip, we should just read from the start
+ }
+
+ LogEntry e;
+ while ((e = readNextEntry()) != null && e.getTimestamp() < endtime) {
+ if (e.getTimestamp() >= starttime && (filter == null || filter.matches(e))) {
+ next = e;
+ return;
+ }
+ skippedAtStart++;
+ }
+ this.filter = filter;
+ }
+
+ synchronized public long size() throws IOException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("size() called");
+ }
+
+ if (this.endtime >= src.getEndTime()) {
+ return src.size() - skippedAtStart;
+ }
+
+ long pos = in.getPosition();
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("saved pos () = " + pos);
+ }
+
+ LogEntry e;
+
+ LogSkipList.Mark lastseg = src.getSkipList().findMarkBefore(this.endtime);
+ in.seek(lastseg.getBytes());
+ buf = ""; // clear the buf so we don't get something we read before we sought
+ // number of entries skipped to get to the end of the iterator, less the number skipped to get to the start
+ long count = lastseg.getEntriesSkipped() - skippedAtStart;
+
+ while ((e = readNextEntry()) != null) {
+ if (LOG.isTraceEnabled()) {
+ //LOG.trace(e);
+ }
+ if (e.getTimestamp() > this.endtime) {
+ break;
+ }
+ count++;
+ }
+ in.seek(pos);
+ buf = "";
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("size() = " + count);
+ }
+
+ return count;
+ }
+
+ synchronized private LogEntry readNextEntry() {
+ try {
+ try {
+ while (true) {
+ String line = in.readLine();
+ if (line == null) {
+ break;
+ }
+
+ Matcher m = src.timep.matcher(line);
+ if (m.lookingAt()) {
+ if (buf.length() > 0) {
+ LogEntry e = new Log4JEntry(src.timestampFromText(dateformat, buf), src.getServerId(), buf);
+ buf = line;
+ return e;
+ }
+ buf = line;
+ } else if (buf.length() > 0) {
+ buf += line + "\n";
+ }
+ }
+ } catch (EOFException eof) {
+ // ignore, we've simply come to the end of the file
+ }
+ if (buf.length() > 0) {
+ LogEntry e = new Log4JEntry(src.timestampFromText(dateformat, buf), src.getServerId(), buf);
+ buf = "";
+ return e;
+ }
+ } catch (Exception e) {
+ LOG.error("Error reading next entry in file (" + src.file + "): " + e);
+ return null;
+ }
+ return null;
+ }
+
+ public boolean hasNext() {
+ return next != null;
+ }
+
+ public LogEntry next() throws NoSuchElementException {
+ LogEntry ret = next;
+ LogEntry e = readNextEntry();
+
+ if (filter != null) {
+ try {
+ while (e != null && !filter.matches(e)) {
+ e = readNextEntry();
+ }
+ } catch (FilterException fe) {
+ throw new NoSuchElementException(e.toString());
+ }
+ }
+
+ if (e != null && e.getTimestamp() < endtime) {
+ next = e;
+ } else {
+ next = null;
+ }
+ return ret;
+ }
+
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("remove not supported for L4J logs");
+ }
+
+ public void close() throws IOException {
+ in.close();
+ }
+
+ public String toString() {
+ String size;
+ try {
+ size = new Long(size()).toString();
+ } catch (IOException ioe) {
+ size = "Unable to read";
+ }
+ return "Log4JSourceIterator(start=" + starttime + ", end=" + endtime + ", size=" + size + ")";
+ }
+ }
+
+ public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException {
+ try {
+ return iterator(starttime, endtime, null);
+ } catch (FilterException fe) {
+ assert(false); //"This should never happen, you can't have a filter exception without a filter");
+ return null;
+ }
+ }
+
+ public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException{
+ // sanitise start and end times
+ if (endtime < starttime) {
+ throw new IllegalArgumentException("End time (" + endtime + ") must be greater or equal to starttime (" + starttime + ")");
+ }
+
+ return new Log4JSourceIterator(this, starttime, endtime, filter);
+ }
+
+ public LogIterator iterator() throws IllegalArgumentException {
+ return iterator(starttime, endtime+1);
+ }
+
+ public Log4JSource(String file) throws IOException {
+ this.file=file;
+
+ timep = Pattern.compile("^(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3})");
+ skiplist = new LogSkipList();
+ init();
+ }
+
+ private static long timestampFromText(SimpleDateFormat format, String s) {
+ Date d = null;
+ try {
+ d = format.parse(s);
+ } catch (ParseException e) {
+ return 0;
+ }
+ Calendar c = new GregorianCalendar();
+ c.setTime(d);
+ return c.getTimeInMillis();
+ }
+
+ private void init() throws IOException {
+ File f = new File(file);
+ RandomAccessFileReader in = new RandomAccessFileReader(f);
+ SimpleDateFormat dateformat = new SimpleDateFormat(DATE_FORMAT);
+ Pattern idp = Pattern.compile("Notification:\\s+\\d+,\\s+\\d+,\\s+\\d+,\\s+(\\d+)");// 1, My\\s+id\\s+=\\s+(\\d+)");
+
+ long lastFp = in.getPosition();
+ String line = in.readLine();
+ Matcher m = null;
+
+ // if we have read data from the file, and it matchs the timep pattern
+ if ((line != null) && (m = timep.matcher(line)).lookingAt()) {
+ starttime = timestampFromText(dateformat, m.group(1));
+ } else {
+ throw new IOException("Invalid log4j format. First line doesn't start with time");
+ }
+
+ /*
+ Count number of log entries. Any line starting with a timestamp counts as an entry
+ */
+ String lastentry = line;
+ try {
+ while (line != null) {
+ m = timep.matcher(line);
+ if (m.lookingAt()) {
+ if (size % skipN == 0) {
+ long time = timestampFromText(dateformat, m.group(1));
+ skiplist.addMark(time, lastFp, size);
+ }
+ size++;
+ lastentry = line;
+ }
+ if (serverid == 0 && (m = idp.matcher(line)).find()) {
+ serverid = Integer.valueOf(m.group(1));
+ }
+
+ lastFp = in.getPosition();
+ line = in.readLine();
+ }
+ } catch (EOFException eof) {
+ // ignore, simply end of file, though really (line!=null) should have caught this
+ } finally {
+ in.close();
+ }
+
+ m = timep.matcher(lastentry);
+ if (m.lookingAt()) {
+ endtime = timestampFromText(dateformat, m.group(1));
+ } else {
+ throw new IOException("Invalid log4j format. Last line doesn't start with time");
+ }
+ }
+
+ public String toString() {
+ return "Log4JSource(file=" + file + ", size=" + size + ", start=" + starttime + ", end=" + endtime +", id=" + serverid +")";
+ }
+
+ public static void main(String[] args) throws IOException {
+ final Log4JSource s = new Log4JSource(args[0]);
+ System.out.println(s);
+
+ LogIterator iter;
+
+ if (args.length == 3) {
+ final long starttime = Long.valueOf(args[1]);
+ final long endtime = Long.valueOf(args[2]);
+ iter = s.iterator(starttime, endtime);
+
+ Thread t1 = new Thread() { public void run () {
+
+ LogIterator iter = s.iterator(starttime, endtime);
+ System.out.println(iter);
+ }; };
+ Thread t2 = new Thread() { public void run () {
+
+ LogIterator iter = s.iterator(starttime, endtime);
+ System.out.println(iter);
+ }; };
+ Thread t3 = new Thread() { public void run () {
+
+ LogIterator iter = s.iterator(starttime, endtime);
+ System.out.println(iter);
+ }; };
+ t1.start();
+ t2.start();
+ // t3.start();
+ } else {
+ iter = s.iterator();
+ }
+
+ /*while (iter.hasNext()) {
+ System.out.println(iter.next());
+ }*/
+ iter.close();
+ }
+
+ public int getServerId() {
+ return serverid;
+ }
+}
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogEntry.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogEntry.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogEntry.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogEntry.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.Serializable;
+import java.util.HashMap;
+
+public abstract class LogEntry implements Serializable {
+ private HashMap attributes;
+
+ public enum Type { UNKNOWN, LOG4J, TXN };
+
+ public LogEntry(long timestamp) {
+ attributes = new HashMap();
+ setAttribute("timestamp", new Long(timestamp));
+ }
+
+ public long getTimestamp() {
+ return (Long)getAttribute("timestamp");
+ }
+
+ public abstract Type getType();
+
+ public void setAttribute(String key, Object v) {
+ attributes.put(key, v);
+ }
+
+ public Object getAttribute(String key) {
+ return attributes.get(key);
+ }
+}
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogIterator.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogIterator.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogIterator.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogIterator.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.Closeable;
+import java.util.Iterator;
+import java.io.IOException;
+
+public interface LogIterator extends Iterator<LogEntry>, Closeable {
+ long size() throws IOException;;
+};
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogServer.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogServer.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogServer.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogServer.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.IOException;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.servlet.ServletException;
+
+import java.io.IOException;
+
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+import org.apache.zookeeper.graph.servlets.*;
+
+public class LogServer extends ServletContextHandler {
+ public LogServer(MergedLogSource src) throws Exception {
+ super(ServletContextHandler.SESSIONS);
+ setContextPath("/");
+
+ addServlet(new ServletHolder(new StaticContent()),"/graph/*");
+
+ addServlet(new ServletHolder(new Fs()),"/fs");
+ addServlet(new ServletHolder(new GraphData(src)), "/data");
+ addServlet(new ServletHolder(new FileLoader(src)), "/loadfile");
+ addServlet(new ServletHolder(new NumEvents(src)), "/info");
+ addServlet(new ServletHolder(new Throughput(src)), "/throughput");
+ }
+
+ public static void main(String[] args) {
+ try {
+ MergedLogSource src = new MergedLogSource(args);
+ System.out.println(src);
+
+ Server server = new Server(8182);
+ server.setHandler(new LogServer(src));
+
+ server.start();
+ server.join();
+
+ } catch (Exception e) {
+ // Something is wrong.
+ e.printStackTrace();
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSkipList.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSkipList.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSkipList.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSkipList.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.NoSuchElementException;
+
+import org.apache.log4j.Logger;
+
+/**
+Generic skip list for holding a rough index of a log file. When the log file is loaded, this
+index is built by adding a mark every n entries. Then when a specific time position is requested
+from the file, a point at most n-1 entries before the time position can be jumped to.
+
+*/
+public class LogSkipList {
+ private static final Logger LOG = Logger.getLogger(LogSkipList.class);
+
+ private LinkedList<Mark> marks;
+
+ public class Mark {
+ private long time;
+ private long bytes;
+ private long skipped;
+
+ public Mark(long time, long bytes, long skipped) {
+ this.time = time;
+ this.bytes = bytes;
+ this.skipped = skipped;
+ }
+
+ public long getTime() { return this.time; }
+ public long getBytes() { return this.bytes; }
+ public long getEntriesSkipped() { return this.skipped; }
+
+ public String toString() {
+ return "Mark(time=" + time + ", bytes=" + bytes + ", skipped=" + skipped + ")";
+ }
+ };
+
+ public LogSkipList() {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("New skip list");
+ }
+ marks = new LinkedList<Mark>();
+ }
+
+ public void addMark(long time, long bytes, long skipped) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("addMark (time:" + time + ", bytes: " + bytes + ", skipped: " + skipped + ")");
+ }
+ marks.add(new Mark(time, bytes, skipped));
+ }
+
+ /**
+ Find the last mark in the skip list before time.
+ */
+ public Mark findMarkBefore(long time) throws NoSuchElementException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("findMarkBefore(" + time + ")");
+ }
+
+ Mark last = marks.getFirst();
+ for (Mark m: marks) {
+ if (m.getTime() > time) {
+ break;
+ }
+ last = m;
+ }
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("return " + last );
+ }
+
+ return last;
+ }
+
+};
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSource.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSource.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSource.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/LogSource.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+import java.util.Iterator;
+
+public interface LogSource extends Iterable<LogEntry> {
+ public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException;
+
+ public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException;
+
+ public LogIterator iterator() throws IllegalArgumentException;
+
+ public boolean overlapsRange(long starttime, long endtime);
+
+ public long size();
+ public long getStartTime();
+ public long getEndTime();
+}
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MeasureThroughput.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MeasureThroughput.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MeasureThroughput.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MeasureThroughput.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.IOException;
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.DataOutputStream;
+import java.io.PrintStream;
+
+import java.util.HashSet;
+
+public class MeasureThroughput {
+ private static final int MS_PER_SEC = 1000;
+ private static final int MS_PER_MIN = MS_PER_SEC*60;
+ private static final int MS_PER_HOUR = MS_PER_MIN*60;
+
+ public static void main(String[] args) throws IOException {
+ MergedLogSource source = new MergedLogSource(args);
+
+ PrintStream ps_ms = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-ms.out")));
+ PrintStream ps_sec = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-sec.out")));
+ PrintStream ps_min = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-min.out")));
+ PrintStream ps_hour = new PrintStream(new BufferedOutputStream(new FileOutputStream("throughput-hour.out")));
+ LogIterator iter;
+
+ System.out.println(source);
+ iter = source.iterator();
+ long currentms = 0;
+ long currentsec = 0;
+ long currentmin = 0;
+ long currenthour = 0;
+ HashSet<Long> zxids_ms = new HashSet<Long>();
+ long zxid_sec = 0;
+ long zxid_min = 0;
+ long zxid_hour = 0;
+
+ while (iter.hasNext()) {
+ LogEntry e = iter.next();
+ TransactionEntry cxn = (TransactionEntry)e;
+
+ long ms = cxn.getTimestamp();
+ long sec = ms/MS_PER_SEC;
+ long min = ms/MS_PER_MIN;
+ long hour = ms/MS_PER_HOUR;
+
+ if (currentms != ms && currentms != 0) {
+ ps_ms.println("" + currentms + " " + zxids_ms.size());
+
+ zxid_sec += zxids_ms.size();
+ zxid_min += zxids_ms.size();
+ zxid_hour += zxids_ms.size();
+ zxids_ms.clear();
+ }
+
+ if (currentsec != sec && currentsec != 0) {
+ ps_sec.println("" + currentsec*MS_PER_SEC + " " + zxid_sec);
+
+ zxid_sec = 0;
+ }
+
+ if (currentmin != min && currentmin != 0) {
+ ps_min.println("" + currentmin*MS_PER_MIN + " " + zxid_min);
+
+ zxid_min = 0;
+ }
+
+ if (currenthour != hour && currenthour != 0) {
+ ps_hour.println("" + currenthour*MS_PER_HOUR + " " + zxid_hour);
+
+ zxid_hour = 0;
+ }
+
+ currentms = ms;
+ currentsec = sec;
+ currentmin = min;
+ currenthour = hour;
+
+ zxids_ms.add(cxn.getZxid());
+ }
+
+ iter.close();
+ ps_ms.close();
+ ps_sec.close();
+ ps_min.close();
+ ps_hour.close();
+ }
+};
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MergedLogSource.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MergedLogSource.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MergedLogSource.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/MergedLogSource.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.util.Date;
+import java.util.zip.Adler32;
+import java.util.zip.Checksum;
+import java.util.HashMap;
+
+import org.apache.jute.BinaryInputArchive;
+import org.apache.jute.InputArchive;
+import org.apache.jute.Record;
+import org.apache.zookeeper.server.TraceFormatter;
+import org.apache.zookeeper.server.persistence.FileHeader;
+import org.apache.zookeeper.server.persistence.FileTxnLog;
+import org.apache.zookeeper.server.util.SerializeUtils;
+import org.apache.zookeeper.txn.TxnHeader;
+
+import org.apache.zookeeper.ZooDefs.OpCode;
+
+import org.apache.zookeeper.txn.CreateSessionTxn;
+import org.apache.zookeeper.txn.CreateTxn;
+import org.apache.zookeeper.txn.DeleteTxn;
+import org.apache.zookeeper.txn.ErrorTxn;
+import org.apache.zookeeper.txn.SetACLTxn;
+import org.apache.zookeeper.txn.SetDataTxn;
+import org.apache.zookeeper.txn.TxnHeader;
+
+import java.io.Closeable;
+import java.io.FileNotFoundException;
+import java.util.Vector;
+import java.util.Iterator;
+import java.util.Collections;
+import java.util.NoSuchElementException;
+
+import org.apache.log4j.Logger;
+
+public class MergedLogSource implements LogSource {
+ private static final Logger LOG = Logger.getLogger(MergedLogSource.class);
+ private Vector<LogSource> sources = null;
+ private long starttime = 0;
+ private long endtime = 0;
+ private long size = 0;
+
+ public boolean overlapsRange(long starttime, long endtime) {
+ return (starttime <= this.endtime && endtime >= this.starttime);
+ }
+
+ public long size() { return size; }
+ public long getStartTime() { return starttime; }
+ public long getEndTime() { return endtime; }
+
+ private class MergedLogSourceIterator implements LogIterator {
+ private LogEntry next = null;
+ private long start = 0;
+ private long end = 0;
+ private MergedLogSource src = null;
+ private LogIterator[] sources = null;
+ private LogEntry[] nexts = null;
+ private FilterOp filter = null;
+
+ public MergedLogSourceIterator(MergedLogSource src, long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException {
+ Vector<LogIterator> iters = new Vector<LogIterator>();
+ for (LogSource s : src.sources) {
+ if (s.overlapsRange(starttime, endtime)) {
+ iters.add(s.iterator(starttime, endtime, filter));
+ }
+ }
+
+ sources = new LogIterator[iters.size()];
+ sources = iters.toArray(sources);
+ nexts = new LogEntry[iters.size()];
+ for (int i = 0; i < sources.length; i++) {
+ if (sources[i].hasNext())
+ nexts[i] = sources[i].next();
+ }
+ this.filter = filter;
+ }
+
+ public MergedLogSourceIterator(MergedLogSource src, long starttime, long endtime) throws IllegalArgumentException, FilterException {
+ this(src, starttime, endtime, null);
+ }
+
+ public long size() throws IOException {
+ long size = 0;
+ for (LogIterator i : sources) {
+ size += i.size();
+ }
+ return size;
+ }
+
+ public boolean hasNext() {
+ for (LogEntry n : nexts) {
+ if (n != null) return true;
+ }
+ return false;
+ }
+
+ public LogEntry next() {
+ int min = -1;
+ for (int i = 0; i < nexts.length; i++) {
+ if (nexts[i] != null) {
+ if (min == -1) {
+ min = i;
+ } else if (nexts[i].getTimestamp() < nexts[min].getTimestamp()) {
+ min = i;
+ }
+ }
+ }
+ if (min == -1) {
+ return null;
+ } else {
+ LogEntry e = nexts[min];
+ nexts[min] = sources[min].next();
+ return e;
+ }
+ }
+
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("remove not supported for Merged logs");
+ }
+
+ public void close() throws IOException {
+ for (LogIterator i : sources) {
+ i.close();
+ }
+ }
+ }
+
+ public LogIterator iterator(long starttime, long endtime) throws IllegalArgumentException {
+ try {
+ return iterator(starttime, endtime, null);
+ } catch (FilterException fe) {
+ assert(false); // shouldn't happen without filter
+ return null;
+ }
+ }
+
+ public LogIterator iterator(long starttime, long endtime, FilterOp filter) throws IllegalArgumentException, FilterException {
+ // sanitise start and end times
+ if (endtime < starttime) {
+ throw new IllegalArgumentException("End time (" + endtime + ") must be greater or equal to starttime (" + starttime + ")");
+ }
+
+ return new MergedLogSourceIterator(this, starttime, endtime, filter);
+ }
+
+ public LogIterator iterator() throws IllegalArgumentException {
+ return iterator(starttime, endtime+1);
+ }
+
+ public MergedLogSource(String[] files) throws IOException {
+ sources = new Vector<LogSource>();
+ for (String f : files) {
+ addSource(f);
+ }
+ }
+
+ public void addSource(String f) throws IOException {
+ LogSource s = null;
+ if (TxnLogSource.isTransactionFile(f)) {
+ s = new TxnLogSource(f);
+ } else {
+ s = new Log4JSource(f);
+ }
+
+ size += s.size();
+ endtime = s.getEndTime() > endtime ? s.getEndTime() : endtime;
+ starttime = s.getStartTime() < starttime || starttime == 0 ? s.getStartTime() : starttime;
+ sources.add(s);
+ }
+
+ public String toString() {
+ String s = "MergedLogSource(size=" + size + ", start=" + starttime + ", end=" + endtime +")";
+ for (LogSource src : sources) {
+ s += "\n\t- " +src;
+ }
+ return s;
+ }
+
+ public static void main(String[] args) throws IOException {
+ System.out.println("Time: " + System.currentTimeMillis());
+ MergedLogSource s = new MergedLogSource(args);
+ System.out.println(s);
+
+ LogIterator iter;
+
+ iter = s.iterator();
+ System.out.println("Time: " + System.currentTimeMillis());
+ System.out.println("Iterator Size: " + iter.size());
+ System.out.println("Time: " + System.currentTimeMillis());
+ /* while (iter.hasNext()) {
+ System.out.println(iter.next());
+ }*/
+ iter.close();
+ System.out.println("Time: " + System.currentTimeMillis());
+ }
+}
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/RandomAccessFileReader.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/RandomAccessFileReader.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/RandomAccessFileReader.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/RandomAccessFileReader.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,327 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+import java.io.File;
+import java.io.Reader;
+import java.io.IOException;
+import java.io.EOFException;
+import java.io.RandomAccessFile;
+import java.io.FileNotFoundException;
+
+import java.io.DataInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+
+import org.apache.log4j.Logger;
+
+public class RandomAccessFileReader extends Reader implements DataInput {
+ private static final Logger LOG = Logger.getLogger(RandomAccessFileReader.class);
+ private RandomAccessFile file;
+ private byte[] buffer;
+ private int buffersize;
+ private int bufferoffset;
+ private long fileoffset;
+ private long fp;
+
+ private static final int DEFAULT_BUFFER_SIZE = 512*1024; // 512k
+ private int point = 0;
+
+ public RandomAccessFileReader(File f) throws FileNotFoundException {
+ file = new RandomAccessFile(f, "r");
+ if (LOG.isDebugEnabled()) {
+ try {
+ LOG.debug("Opened file(" + f + ") with FD (" + file.getFD() + ")");
+ } catch (IOException ioe) {
+ LOG.debug("Opened file(" + f + ") coulds get FD");
+ }
+ }
+
+ buffer = new byte[DEFAULT_BUFFER_SIZE];
+ buffersize = 0;
+ bufferoffset = 0;
+ fileoffset = 0;
+ fp = 0;
+ }
+
+ /**
+ fill the buffer from the file.
+ fp keeps track of the file pointer.
+ fileoffset is the offset into the file to where the buffer came from.
+ */
+ private int fill() throws IOException {
+ fileoffset = fp;
+ int read = file.read(buffer, 0, buffer.length);
+
+ if (LOG.isDebugEnabled()) {
+ String buf = new String(buffer, 0, 40, "UTF-8");
+ LOG.debug("fill(buffer=" + buf + ")");
+ }
+
+ if (read == -1) { // eof reached
+ buffersize = 0;
+ } else {
+ buffersize = read;
+ }
+ fp += buffersize;
+ bufferoffset = 0;
+
+ return buffersize;
+ }
+
+ /**
+ * Reader interface
+ */
+ public boolean markSupported() { return false; }
+
+ /**
+ copy what we can from buffer. if it's not enough, fill buffer again and copy again
+ */
+ synchronized public int read(char[] cbuf, int off, int len) throws IOException {
+ // This could be faster, but probably wont be used
+ byte[] b = new byte[2];
+ int bytesread = 0;
+ while (len > 0) {
+ int read = read(b, 0, 2);
+ bytesread += read;
+ if (read < 2) {
+ return bytesread;
+ }
+ cbuf[off] = (char)((b[0] << 8) | (b[1] & 0xff));
+ off += read;
+ len -= read;
+ }
+
+ return bytesread;
+ }
+
+ synchronized public int read(byte[] buf, int off, int len) throws IOException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read(buf, off=" + off + ", len=" + len);
+ }
+
+ int read = 0;
+ while (len > 0) {
+ if (buffersize == 0) {
+ fill();
+ if (buffersize == 0) {
+ break;
+ }
+ }
+
+ int tocopy = Math.min(len, buffersize);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("tocopy=" + tocopy);
+ }
+
+ System.arraycopy(buffer, bufferoffset, buf, off, tocopy);
+ buffersize -= tocopy;
+ bufferoffset += tocopy;
+
+ len -= tocopy;
+ read += tocopy;
+ off += tocopy;
+ }
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("read=" + read);
+ }
+
+ return read;
+ }
+
+ public void close() throws IOException {
+ file.close();
+ }
+
+ /**
+ * Seek interface
+ */
+ public long getPosition() {
+ return bufferoffset + fileoffset;
+ }
+
+ synchronized public void seek(long pos) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("seek(" + pos + ")");
+ }
+ file.seek(pos);
+ fp = pos;
+ buffersize = 0; // force a buffer fill on next read
+ }
+
+ /**
+ works like the usual readLine but disregards \r to make things easier
+ */
+ synchronized public String readLine() throws IOException {
+ StringBuffer s = null;
+
+ // go through buffer until i find a \n, if i reach end of buffer first, put whats in buffer into string buffer,
+ // repeat
+ buffering:
+ for (;;) {
+ if (buffersize == 0) {
+ fill();
+ if (buffersize == 0) {
+ break;
+ }
+ }
+
+ for (int i = 0; i < buffersize; i++) {
+ if (buffer[bufferoffset + i] == '\n') {
+ if (i > 0) { // if \n is first char in buffer, leave the string buffer empty
+ if (s == null) { s = new StringBuffer(); }
+ s.append(new String(buffer, bufferoffset, i, "UTF-8"));
+ }
+ bufferoffset += i+1;
+ buffersize -= i+1;
+ break buffering;
+ }
+ }
+
+ // We didn't find \n, read the whole buffer into string buffer
+ if (s == null) { s = new StringBuffer(); }
+ s.append(new String(buffer, bufferoffset, buffersize, "UTF-8"));
+ buffersize = 0;
+ }
+
+ if (s == null) {
+ return null;
+ } else {
+ return s.toString();
+ }
+ }
+
+ /**
+ DataInput interface
+ */
+ public void readFully(byte[] b) throws IOException {
+ readFully(b, 0, b.length);
+ }
+
+ public void readFully(byte[] b, int off, int len) throws IOException
+ {
+ while (len > 0) {
+ int read = read(b, off, len);
+ len -= read;
+ off += read;
+
+ if (read == 0) {
+ throw new EOFException("End of file reached");
+ }
+ }
+ }
+
+ public int skipBytes(int n) throws IOException {
+ seek(getPosition() + n);
+ return n;
+ }
+
+ public boolean readBoolean() throws IOException {
+ return (readByte() != 0);
+ }
+
+ public byte readByte() throws IOException {
+ byte[] b = new byte[1];
+ readFully(b, 0, 1);
+ return b[0];
+ }
+
+ public int readUnsignedByte() throws IOException {
+ return (int)readByte();
+ }
+
+ public short readShort() throws IOException {
+ byte[] b = new byte[2];
+ readFully(b, 0, 2);
+ return (short)((b[0] << 8) | (b[1] & 0xff));
+ }
+
+ public int readUnsignedShort() throws IOException {
+ byte[] b = new byte[2];
+ readFully(b, 0, 2);
+ return (((b[0] & 0xff) << 8) | (b[1] & 0xff));
+ }
+
+ public char readChar() throws IOException {
+ return (char)readShort();
+ }
+
+ public int readInt() throws IOException {
+ byte[] b = new byte[4];
+ readFully(b, 0, 4);
+ return (((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) | ((b[2] & 0xff) << 8) | (b[3] & 0xff));
+ }
+
+ public long readLong() throws IOException {
+ byte[] b = new byte[8];
+ readFully(b, 0, 8);
+
+ return (((long)(b[0] & 0xff) << 56) | ((long)(b[1] & 0xff) << 48) |
+ ((long)(b[2] & 0xff) << 40) | ((long)(b[3] & 0xff) << 32) |
+ ((long)(b[4] & 0xff) << 24) | ((long)(b[5] & 0xff) << 16) |
+ ((long)(b[6] & 0xff) << 8) | ((long)(b[7] & 0xff)));
+ }
+
+ public float readFloat() throws IOException {
+ return Float.intBitsToFloat(readInt());
+ }
+
+ public double readDouble() throws IOException {
+ return Double.longBitsToDouble(readLong());
+ }
+
+ public String readUTF() throws IOException {
+ int len = readUnsignedShort();
+ byte[] bytes = new byte[len+2];
+ bytes[0] = (byte)((len >> 8) & 0xFF);
+ bytes[1] = (byte)(len & 0xFF);
+ readFully(bytes, 2, len);
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(bytes));
+ return dis.readUTF();
+ }
+
+ public static void main(String[] args) throws IOException {
+ RandomAccessFileReader f = new RandomAccessFileReader(new File(args[0]));
+
+ long pos0 = f.getPosition();
+ for (int i = 0; i < 5; i++) {
+ System.out.println(f.readLine());
+ }
+ System.out.println("=============");
+ long pos1 = f.getPosition();
+ System.out.println("pos: " + pos1);
+ for (int i = 0; i < 5; i++) {
+ System.out.println(f.readLine());
+ }
+ System.out.println("=============");
+ f.seek(pos1);
+ for (int i = 0; i < 5; i++) {
+ System.out.println(f.readLine());
+ }
+ System.out.println("=============");
+ f.seek(pos0);
+ for (int i = 0; i < 5; i++) {
+ System.out.println(f.readLine());
+ }
+ long pos2 = f.getPosition();
+ System.out.println("=============");
+ System.out.println(f.readLine());
+ f.seek(pos2);
+ System.out.println(f.readLine());
+ }
+};
\ No newline at end of file
Added: hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TransactionEntry.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TransactionEntry.java?rev=953041&view=auto
==============================================================================
--- hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TransactionEntry.java (added)
+++ hadoop/zookeeper/trunk/src/contrib/loggraph/src/java/org/apache/zookeeper/graph/TransactionEntry.java Wed Jun 9 15:26:22 2010
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.graph;
+
+public class TransactionEntry extends LogEntry {
+ public TransactionEntry(long timestamp, long clientId, long Cxid, long Zxid, String op) {
+ this(timestamp, clientId, Cxid, Zxid, op, "");
+ }
+
+ public TransactionEntry(long timestamp, long clientId, long Cxid, long Zxid, String op, String extra) {
+ super(timestamp);
+ setAttribute("client-id", new Long(clientId));
+ setAttribute("cxid", new Long(Cxid));
+ setAttribute("zxid", new Long(Zxid));
+ setAttribute("operation", op);
+ setAttribute("extra", extra);
+ }
+
+ public long getClientId() {
+ return (Long)getAttribute("client-id");
+ }
+
+ public long getCxid() {
+ return (Long)getAttribute("cxid");
+ }
+
+ public long getZxid() {
+ return (Long)getAttribute("zxid");
+ }
+
+ public String getOp() {
+ return (String)getAttribute("operation");
+ }
+
+ public String getExtra() {
+ return (String)getAttribute("extra");
+ }
+
+ public String toString() {
+ return getTimestamp() + ":::session(0x" + Long.toHexString(getClientId()) + ") cxid(0x" + Long.toHexString(getCxid()) + ") zxid(0x" + Long.toHexString(getZxid()) + ") op(" + getOp() + ") extra(" + getExtra() +")";
+ }
+
+ public Type getType() { return LogEntry.Type.TXN; }
+}
\ No newline at end of file