You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by ma...@apache.org on 2016/10/12 08:06:24 UTC
incubator-gearpump git commit: [GEARPUMP-222] remove DFSJarStore from
gearpump-daemon
Repository: incubator-gearpump
Updated Branches:
refs/heads/master 1b9889121 -> a01809b25
[GEARPUMP-222] remove DFSJarStore from gearpump-daemon
Author: huafengw <fv...@gmail.com>
Closes #96 from huafengw/jarclass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/a01809b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/a01809b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/a01809b2
Branch: refs/heads/master
Commit: a01809b2588a5dd1038b7afa8dca5148729e634c
Parents: 1b98891
Author: huafengw <fv...@gmail.com>
Authored: Wed Oct 12 16:06:15 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Wed Oct 12 16:06:15 2016 +0800
----------------------------------------------------------------------
.../org.apache.gearpump.jarstore.JarStore | 19 ++++++
.../org.apache.gearpump.jarstore.JarStore | 19 ++++++
.../org.apache.gearpump.jarstore.JarStore | 20 ------
.../gearpump/jarstore/dfs/DFSJarStore.scala | 67 --------------------
.../org.apache.gearpump.jarstore.JarStore | 20 ------
.../gearpump/cluster/utils/SystemOperation.java | 7 +-
.../org.apache.gearpump.jarstore.JarStore | 19 ++++++
.../gearpump/jarstore/dfs/DFSJarStore.scala | 66 +++++++++++++++++++
project/Build.scala | 23 +++++--
project/Pack.scala | 3 +-
10 files changed, 145 insertions(+), 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
----------------------------------------------------------------------
diff --git a/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
new file mode 100644
index 0000000..c64d444
--- /dev/null
+++ b/core/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gearpump.jarstore.local.LocalJarStore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
----------------------------------------------------------------------
diff --git a/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
new file mode 100644
index 0000000..c64d444
--- /dev/null
+++ b/core/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gearpump.jarstore.local.LocalJarStore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
----------------------------------------------------------------------
diff --git a/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
deleted file mode 100644
index e173a8a..0000000
--- a/daemon/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-org.apache.gearpump.jarstore.local.LocalJarStore
-org.apache.gearpump.jarstore.dfs.DFSJarStore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
----------------------------------------------------------------------
diff --git a/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala b/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
deleted file mode 100644
index ebaf354..0000000
--- a/daemon/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gearpump.jarstore.dfs
-
-import java.io.{InputStream, OutputStream}
-
-import com.typesafe.config.Config
-import org.apache.gearpump.jarstore.JarStore
-import org.apache.gearpump.util.Constants
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
-
-/**
- * DFSJarStore store the uploaded jar on HDFS
- */
-class DFSJarStore extends JarStore {
- private var rootPath: Path = null
- override val scheme: String = "hdfs"
-
- override def init(config: Config): Unit = {
- rootPath = new Path(config.getString(Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH))
- val fs = rootPath.getFileSystem(new Configuration())
- if (!fs.exists(rootPath)) {
- fs.mkdirs(rootPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
- }
- }
-
- /**
- * Creates the file on JarStore.
- *
- * @param fileName name of the file to be created on JarStore.
- * @return OutputStream returns a stream into which the data can be written.
- */
- override def createFile(fileName: String): OutputStream = {
- val filePath = new Path(rootPath, fileName)
- val fs = filePath.getFileSystem(new Configuration())
- fs.create(filePath)
- }
-
- /**
- * Gets the InputStream to read the file
- *
- * @param fileName name of the file to be read on JarStore.
- * @return InputStream returns a stream from which the data can be read.
- */
- override def getFile(fileName: String): InputStream = {
- val filePath = new Path(rootPath, fileName)
- val fs = filePath.getFileSystem(new Configuration())
- fs.open(filePath)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
----------------------------------------------------------------------
diff --git a/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
deleted file mode 100644
index e173a8a..0000000
--- a/daemon/src/test/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-org.apache.gearpump.jarstore.local.LocalJarStore
-org.apache.gearpump.jarstore.dfs.DFSJarStore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
----------------------------------------------------------------------
diff --git a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
index 5b2a890..4983dde 100644
--- a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
+++ b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
@@ -17,11 +17,12 @@
*/
package org.apache.gearpump.cluster.utils;
-import org.apache.commons.io.IOUtils;
+import org.apache.gearpump.google.common.io.CharStreams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.io.InputStreamReader;
public class SystemOperation {
@@ -44,8 +45,8 @@ public class SystemOperation {
Process process = new ProcessBuilder(new String[]{"/bin/bash", "-c", cmd}).start();
try {
process.waitFor();
- String output = IOUtils.toString(process.getInputStream());
- String errorOutput = IOUtils.toString(process.getErrorStream());
+ String output = CharStreams.toString(new InputStreamReader(process.getInputStream()));
+ String errorOutput = CharStreams.toString(new InputStreamReader(process.getErrorStream()));
LOG.debug("Shell Output: " + output);
if (errorOutput.length() != 0) {
LOG.error("Shell Error Output: " + errorOutput);
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
----------------------------------------------------------------------
diff --git a/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
new file mode 100644
index 0000000..2809247
--- /dev/null
+++ b/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gearpump.jarstore.dfs.DFSJarStore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
----------------------------------------------------------------------
diff --git a/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala b/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
new file mode 100644
index 0000000..ae4cf46
--- /dev/null
+++ b/gearpump-hadoop/src/main/scala/org/apache/gearpump/jarstore/dfs/DFSJarStore.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gearpump.jarstore.dfs
+
+import java.io.{InputStream, OutputStream}
+import org.apache.gearpump.util.Constants
+import org.apache.gearpump.jarstore.JarStore
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import com.typesafe.config.Config
+import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
+
+/**
+ * DFSJarStore store the uploaded jar on HDFS
+ */
+class DFSJarStore extends JarStore {
+ private var rootPath: Path = null
+ override val scheme: String = "hdfs"
+
+ override def init(config: Config): Unit = {
+ rootPath = new Path(config.getString(Constants.GEARPUMP_APP_JAR_STORE_ROOT_PATH))
+ val fs = rootPath.getFileSystem(new Configuration())
+ if (!fs.exists(rootPath)) {
+ fs.mkdirs(rootPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL))
+ }
+ }
+
+ /**
+ * Creates the file on JarStore.
+ *
+ * @param fileName name of the file to be created on JarStore.
+ * @return OutputStream returns a stream into which the data can be written.
+ */
+ override def createFile(fileName: String): OutputStream = {
+ val filePath = new Path(rootPath, fileName)
+ val fs = filePath.getFileSystem(new Configuration())
+ fs.create(filePath)
+ }
+
+ /**
+ * Gets the InputStream to read the file
+ *
+ * @param fileName name of the file to be read on JarStore.
+ * @return InputStream returns a stream from which the data can be read.
+ */
+ override def getFile(fileName: String): InputStream = {
+ val filePath = new Path(rootPath, fileName)
+ val fs = filePath.getFileSystem(new Configuration())
+ fs.open(filePath)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/project/Build.scala
----------------------------------------------------------------------
diff --git a/project/Build.scala b/project/Build.scala
index 34f0ae2..4552a64 100644
--- a/project/Build.scala
+++ b/project/Build.scala
@@ -145,8 +145,7 @@ object Build extends sbt.Build {
"com.typesafe.akka" %% "akka-cluster" % akkaVersion,
"com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion,
"commons-logging" % "commons-logging" % commonsLoggingVersion,
- "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion,
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided"
+ "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion
)
)
@@ -258,7 +257,7 @@ object Build extends sbt.Build {
base = file("."),
settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting)
.aggregate(shaded, core, daemon, streaming, services, external_kafka, external_monoid,
- external_serializer, examples, storm, yarn, external_hbase, packProject,
+ external_serializer, examples, storm, yarn, external_hbase, gearpumpHadoop, packProject,
external_hadoopfs, integration_test).settings(Defaults.itSettings: _*)
.disablePlugins(sbtassembly.AssemblyPlugin)
@@ -463,14 +462,24 @@ object Build extends sbt.Build {
.dependsOn (streaming % "test->test; compile->compile")
.disablePlugins(sbtassembly.AssemblyPlugin)
+ lazy val gearpumpHadoop = Project(
+ id = "gearpump-hadoop",
+ base = file("gearpump-hadoop"),
+ settings = commonSettings ++ noPublish ++
+ Seq(
+ libraryDependencies ++= Seq(
+ "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
+ "org.apache.hadoop" % "hadoop-common" % hadoopVersion
+ )
+ )
+ ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin)
+
lazy val yarn = Project(
id = "gearpump-experiments-yarn",
base = file("experiments/yarn"),
settings = commonSettings ++ noPublish ++
Seq(
libraryDependencies ++= Seq(
- "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-common" % hadoopVersion,
"org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion,
"org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion,
"org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion,
@@ -480,8 +489,8 @@ object Build extends sbt.Build {
"org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided"
)
))
- .dependsOn(services % "test->test;compile->compile", daemon % "provided", core % "provided")
- .disablePlugins(sbtassembly.AssemblyPlugin)
+ .dependsOn(services % "test->test;compile->compile", daemon % "provided",
+ core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin)
lazy val external_hbase = Project(
id = "gearpump-external-hbase",
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/a01809b2/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index 1c87653..13e53de 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -114,7 +114,8 @@ object Pack extends sbt.Build {
packLibDir := Map(
"lib" -> new ProjectsToPack(core.id, streaming.id),
"lib/daemon" -> new ProjectsToPack(daemon.id, cgroup.id).exclude(core.id, streaming.id),
- "lib/yarn" -> new ProjectsToPack(yarn.id).exclude(services.id, daemon.id),
+ "lib/yarn" -> new ProjectsToPack(gearpumpHadoop.id, yarn.id).
+ exclude(services.id, daemon.id, core.id),
"lib/services" -> new ProjectsToPack(services.id).exclude(daemon.id),
"lib/storm" -> new ProjectsToPack(storm.id).exclude(streaming.id)
),