You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2013/01/26 07:04:36 UTC
[1/28] git commit: Re-applied old patch from KAFKA-139
Re-applied old patch from KAFKA-139
Had to make some additional changes based on further mainline work. Leaving
KafkaProject.scala in place as a reference for now.
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/45cb03d4
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/45cb03d4
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/45cb03d4
Branch: refs/heads/trunk
Commit: 45cb03d456c3c541815691e80f5216a6e524c95c
Parents: 3244bca
Author: Derek Chen-Becker <de...@precog.com>
Authored: Fri Dec 14 17:12:38 2012 -0700
Committer: Derek Chen-Becker <de...@precog.com>
Committed: Fri Dec 14 17:12:38 2012 -0700
----------------------------------------------------------------------
.gitignore | 2 +
README.md | 21 +++--
bin/run-rat.sh | 2 +-
contrib/hadoop-consumer/build.sbt | 1 +
contrib/hadoop-producer/build.sbt | 1 +
core/build.sbt | 24 ++++++
examples/build.sbt | 3 +
lib/apache-rat-0.8.jar | Bin 0 -> 1165578 bytes
project/Build.scala | 143 ++++++++++++++++++++++++++++++++
project/build.properties | 10 +--
project/plugins.sbt | 5 +
project/plugins/Plugins.scala | 23 -----
12 files changed, 193 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 1fc794d..553a077 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,5 @@ project/sbt_project_definition.iml
*~
*#
.#*
+rat.out
+TAGS
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index a028bce..5bb7a80 100644
--- a/README.md
+++ b/README.md
@@ -19,21 +19,21 @@ The Kafka code is available from svn or a read only git mirror:
* svn co http://svn.apache.org/repos/asf/incubator/kafka/trunk kafka
* git clone git://git.apache.org/kafka.git
-To build:
+To build for all supported versions of Scala:
-1. ./sbt
-2. update - This downloads all the dependencies for all sub projects
-3. package - This will compile all sub projects and creates all the jars
+1. ./sbt +package
-Here are some useful sbt commands, to be executed at the sbt command prompt (./sbt) -
+To build for a particular version of Scala (either 2.8.0 or 2.9.1):
-actions : Lists all the sbt commands and their descriptions
+1. ./sbt "++ 2.8.0 package" *or* ./sbt "++ 2.9.1 package"
-clean : Deletes all generated files (the target directory).
+Here are some useful sbt commands, to be executed at the sbt command prompt (./sbt). Prefixing with "++<version> " runs the
+command for a specific Scala version, prefixing with "+" will perform the action for all versions of Scala, and no prefix
+runs the command for the default (2.8.0) version of Scala. -
-clean-cache : Deletes the cache of artifacts downloaded for automatically managed dependencies.
+tasks : Lists all the sbt commands and their descriptions
-clean-lib : Deletes the managed library directory.
+clean : Deletes all generated files (the target directory).
compile : Compile all the sub projects, but not create the jars
@@ -41,7 +41,7 @@ test : Run all unit tests in all sub projects
release-zip : Create all the jars, run unit tests and create a deployable release zip
-package-all: Creates jars for src, test, docs etc
+package: Creates jars for src, test, docs etc
projects : List all the sub projects
@@ -53,4 +53,5 @@ test-only package.test.TestName : Runs only the specified test in the current su
run : Provides options to run any of the classes that have a main method. For example, you can switch to project java-examples, and run the examples there by executing "project java-examples" followed by "run"
+For more details please see the [SBT documentation](https://github.com/harrah/xsbt/wiki)
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/bin/run-rat.sh
----------------------------------------------------------------------
diff --git a/bin/run-rat.sh b/bin/run-rat.sh
index 28c0ccd..1b7bc31 100755
--- a/bin/run-rat.sh
+++ b/bin/run-rat.sh
@@ -23,7 +23,7 @@ else
JAVA="$JAVA_HOME/bin/java"
fi
-rat_command="$JAVA -jar $base_dir/lib_managed/scala_2.8.0/compile/apache-rat-0.8.jar --dir $base_dir "
+rat_command="$JAVA -jar $base_dir/lib/apache-rat-0.8.jar --dir $base_dir "
for f in $(cat $rat_excludes_file);
do
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/contrib/hadoop-consumer/build.sbt
----------------------------------------------------------------------
diff --git a/contrib/hadoop-consumer/build.sbt b/contrib/hadoop-consumer/build.sbt
new file mode 100644
index 0000000..02e95eb
--- /dev/null
+++ b/contrib/hadoop-consumer/build.sbt
@@ -0,0 +1 @@
+crossPaths := false
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/contrib/hadoop-producer/build.sbt
----------------------------------------------------------------------
diff --git a/contrib/hadoop-producer/build.sbt b/contrib/hadoop-producer/build.sbt
new file mode 100644
index 0000000..02e95eb
--- /dev/null
+++ b/contrib/hadoop-producer/build.sbt
@@ -0,0 +1 @@
+crossPaths := false
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/core/build.sbt
----------------------------------------------------------------------
diff --git a/core/build.sbt b/core/build.sbt
new file mode 100644
index 0000000..df58410
--- /dev/null
+++ b/core/build.sbt
@@ -0,0 +1,24 @@
+import sbt._
+import Keys._
+
+name := "kafka"
+
+resolvers ++= Seq(
+ "SonaType ScalaTest repo" at "https://oss.sonatype.org/content/groups/public/org/scalatest/"
+)
+
+libraryDependencies ++= Seq(
+ "org.apache.zookeeper" % "zookeeper" % "3.3.4",
+ "com.github.sgroschupf" % "zkclient" % "0.1",
+ "org.xerial.snappy" % "snappy-java" % "1.0.4.1",
+ "org.easymock" % "easymock" % "3.0" % "test",
+ "junit" % "junit" % "4.1" % "test"
+)
+
+libraryDependencies <<= (scalaVersion, libraryDependencies) { (sv, deps) =>
+ deps :+ (sv match {
+ case "2.8.0" => "org.scalatest" % "scalatest" % "1.2" % "test"
+ case _ => "org.scalatest" %% "scalatest" % "1.8" % "test"
+ })
+}
+
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/examples/build.sbt
----------------------------------------------------------------------
diff --git a/examples/build.sbt b/examples/build.sbt
new file mode 100644
index 0000000..d12d701
--- /dev/null
+++ b/examples/build.sbt
@@ -0,0 +1,3 @@
+name := "kafka-java-examples"
+
+crossPaths := false
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/lib/apache-rat-0.8.jar
----------------------------------------------------------------------
diff --git a/lib/apache-rat-0.8.jar b/lib/apache-rat-0.8.jar
new file mode 100644
index 0000000..bdc4372
Binary files /dev/null and b/lib/apache-rat-0.8.jar differ
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
new file mode 100644
index 0000000..a81f191
--- /dev/null
+++ b/project/Build.scala
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import sbt._
+import Keys._
+import java.io.File
+
+import scala.xml.{Node, Elem}
+import scala.xml.transform.{RewriteRule, RuleTransformer}
+
+object KafkaBuild extends Build {
+ val commonSettings = Seq(
+ version := "0.8.0",
+ organization := "org.apache.kafka",
+ scalacOptions ++= Seq("-deprecation", "-unchecked", "-g:none"),
+ crossScalaVersions := Seq("2.8.0", "2.9.1", "2.9.2"),
+ scalaVersion := "2.8.0",
+ javacOptions ++= Seq("-Xlint:unchecked", "-source", "1.5"),
+ parallelExecution in Test := false, // Prevent tests from overrunning each other
+ libraryDependencies ++= Seq(
+ "log4j" % "log4j" % "1.2.15",
+ "net.sf.jopt-simple" % "jopt-simple" % "3.2",
+ "org.slf4j" % "slf4j-simple" % "latest.release"
+ ),
+ // The issue is going from log4j 1.2.14 to 1.2.15, the developers added some features which required
+ // some dependencies on various sun and javax packages.
+ ivyXML := <dependencies>
+ <exclude module="javax"/>
+ <exclude module="jmxri"/>
+ <exclude module="jmxtools"/>
+ <exclude module="mail"/>
+ <exclude module="jms"/>
+ <dependency org="org.apache.zookeeper" name="zookeeper" rev="3.3.4">
+ <exclude org="log4j" module="log4j"/>
+ <exclude org="jline" module="jline"/>
+ </dependency>
+ </dependencies>
+ )
+
+ val hadoopSettings = Seq(
+ javacOptions ++= Seq("-Xlint:deprecation"),
+ libraryDependencies ++= Seq(
+ "org.apache.avro" % "avro" % "1.4.0",
+ "org.apache.pig" % "pig" % "0.8.0",
+ "commons-logging" % "commons-logging" % "1.0.4",
+ "org.codehaus.jackson" % "jackson-core-asl" % "1.5.5",
+ "org.codehaus.jackson" % "jackson-mapper-asl" % "1.5.5",
+ "org.apache.hadoop" % "hadoop-core" % "0.20.2"
+ ),
+ ivyXML :=
+ <dependencies>
+ <exclude module="netty"/>
+ <exclude module="javax"/>
+ <exclude module="jmxri"/>
+ <exclude module="jmxtools"/>
+ <exclude module="mail"/>
+ <exclude module="jms"/>
+ <dependency org="org.apache.hadoop" name="hadoop-core" rev="0.20.2">
+ <exclude org="junit" module="junit"/>
+ </dependency>
+ <dependency org="org.apache.pig" name="pig" rev="0.8.0">
+ <exclude org="junit" module="junit"/>
+ </dependency>
+ </dependencies>
+ )
+
+ val coreSettings = Seq(
+ pomPostProcess := { (pom: Node) => MetricsDepAdder(ZkClientDepAdder(pom)) }
+ )
+
+ val runRat = TaskKey[Unit]("run-rat-task", "Runs Apache rat on Kafka")
+ val runRatTask = runRat := {
+ "bin/run-rat.sh" !
+ }
+
+ lazy val kafka = Project(id = "Kafka", base = file(".")).aggregate(core, examples, contrib).settings((commonSettings ++ runRatTask): _*)
+ lazy val core = Project(id = "core", base = file("core")).settings(commonSettings: _*).settings(coreSettings: _*)
+ lazy val examples = Project(id = "java-examples", base = file("examples")).settings(commonSettings :_*) dependsOn (core)
+ lazy val perf = Project(id = "perf", base = file("perf")).settings((Seq(name := "kafka-perf") ++ commonSettings):_*) dependsOn (core)
+
+ lazy val contrib = Project(id = "contrib", base = file("contrib")).aggregate(hadoopProducer, hadoopConsumer).settings(commonSettings :_*)
+ lazy val hadoopProducer = Project(id = "hadoop-producer", base = file("contrib/hadoop-producer")).settings(hadoopSettings ++ commonSettings: _*) dependsOn (core)
+ lazy val hadoopConsumer = Project(id = "hadoop-consumer", base = file("contrib/hadoop-consumer")).settings(hadoopSettings ++ commonSettings: _*) dependsOn (core)
+
+
+ // POM Tweaking for core:
+ def zkClientDep =
+ <dependency>
+ <groupId>zkclient</groupId>
+ <artifactId>zkclient</artifactId>
+ <version>20120522</version>
+ <scope>compile</scope>
+ </dependency>
+
+ def metricsDeps =
+ <dependencies>
+ <dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>3.0.0-c0c8be71</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.yammer.metrics</groupId>
+ <artifactId>metrics-annotations</artifactId>
+ <version>3.0.0-c0c8be71</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ object ZkClientDepAdder extends RuleTransformer(new RewriteRule() {
+ override def transform(node: Node): Seq[Node] = node match {
+ case Elem(prefix, "dependencies", attribs, scope, deps @ _*) => {
+ Elem(prefix, "dependencies", attribs, scope, deps ++ zkClientDep:_*)
+ }
+ case other => other
+ }
+ })
+
+ object MetricsDepAdder extends RuleTransformer(new RewriteRule() {
+ override def transform(node: Node): Seq[Node] = node match {
+ case Elem(prefix, "dependencies", attribs, scope, deps @ _*) => {
+ Elem(prefix, "dependencies", attribs, scope, deps ++ metricsDeps:_*)
+ }
+ case other => other
+ }
+ })
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/project/build.properties
----------------------------------------------------------------------
diff --git a/project/build.properties b/project/build.properties
index a8895d3..d0f1e12 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -14,11 +14,5 @@
# limitations under the License.
#Project properties
#Mon Feb 28 11:55:49 PST 2011
-project.name=Kafka
-sbt.version=0.7.5
-project.version=0.8.0
-build.scala.versions=2.8.0
-contrib.root.dir=contrib
-lib.dir=lib
-target.dir=target/scala_2.8.0
-dist.dir=dist
+sbt.version=0.12.1
+
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/project/plugins.sbt b/project/plugins.sbt
new file mode 100644
index 0000000..48d44c8
--- /dev/null
+++ b/project/plugins.sbt
@@ -0,0 +1,5 @@
+resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.8.5")
+
+addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.2.0")
http://git-wip-us.apache.org/repos/asf/kafka/blob/45cb03d4/project/plugins/Plugins.scala
----------------------------------------------------------------------
diff --git a/project/plugins/Plugins.scala b/project/plugins/Plugins.scala
deleted file mode 100644
index 0777d82..0000000
--- a/project/plugins/Plugins.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import sbt._
-
-class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
- val repo = "GH-pages repo" at "http://mpeltonen.github.com/maven/"
- val idea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.1-SNAPSHOT"
-}