You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2018/11/27 14:15:19 UTC
[GitHub] GJL closed pull request #7164: [FLINK-10992][tests] Revise Hadoop
configuration.
GJL closed pull request #7164: [FLINK-10992][tests] Revise Hadoop configuration.
URL: https://github.com/apache/flink/pull/7164
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/flink-jepsen/src/jepsen/flink/hadoop.clj b/flink-jepsen/src/jepsen/flink/hadoop.clj
index 99f3f72d45c..4ba78593b44 100644
--- a/flink-jepsen/src/jepsen/flink/hadoop.clj
+++ b/flink-jepsen/src/jepsen/flink/hadoop.clj
@@ -19,12 +19,15 @@
[clojure.tools.logging :refer :all]
[jepsen
[control :as c]
- [db :as db]]
- [jepsen.control.util :as cu]))
+ [db :as db]
+ [util :refer [meh]]]
+ [jepsen.control.util :as cu]
+ [jepsen.flink.utils :as fu]))
(def install-dir "/opt/hadoop")
(def hadoop-conf-dir (str install-dir "/etc/hadoop"))
-(def yarn-log-dir "/tmp/logs/yarn")
+(def log-dir (str install-dir "/logs"))
+(def yarn-log-dir (str log-dir "/yarn"))
(defn name-node
[nodes]
@@ -51,7 +54,12 @@
(defn core-site-config
[test]
- {:fs.defaultFS (str "hdfs://" (name-node (:nodes test)) ":9000")})
+ {:hadoop.tmp.dir (str install-dir "/tmp")
+ :fs.defaultFS (str "hdfs://" (name-node (:nodes test)) ":9000")})
+
+(defn hdfs-site-config
+ [_]
+ {:dfs.replication "1"})
(defn property-value
[property value]
@@ -66,8 +74,23 @@
(xml/element :configuration
{}
(map (fn [[k v]] (property-value k v)) (seq config))))]
- (c/exec :echo config-xml :> config-file)
- ))
+ (c/exec :echo config-xml :> config-file)))
+
+(defn- write-hadoop-env!
+ "Configures additional environment variables in hadoop-env.sh"
+ []
+ (let [env-vars ["export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
+ "export HADOOP_NAMENODE_OPTS=\"-Xms2G -Xmx2G $HADOOP_NAMENODE_OPTS\""
+ "export HADOOP_DATANODE_OPTS=\"-Xms2G -Xmx2G $HADOOP_DATANODE_OPTS\""]]
+ (doseq [env-var env-vars]
+ (c/exec :echo env-var :>> (str install-dir "/etc/hadoop/hadoop-env.sh")))))
+
+(defn- write-configuration!
+ [test]
+ (write-config! (str install-dir "/etc/hadoop/yarn-site.xml") (yarn-site-config test))
+ (write-config! (str install-dir "/etc/hadoop/core-site.xml") (core-site-config test))
+ (write-config! (str install-dir "/etc/hadoop/hdfs-site.xml") (hdfs-site-config test))
+ (write-hadoop-env!))
(defn start-name-node!
[test node]
@@ -104,12 +127,6 @@
(info "Start NodeManager")
(c/exec (str install-dir "/sbin/yarn-daemon.sh") :--config hadoop-conf-dir :start :nodemanager)))
-(defn find-files!
- [dir]
- (->>
- (clojure.string/split (c/exec :find dir :-type :f) #"\n")
- (remove clojure.string/blank?)))
-
(defn db
[url]
(reify db/DB
@@ -117,26 +134,20 @@
(info "Install Hadoop from" url)
(c/su
(cu/install-archive! url install-dir)
- (write-config! (str install-dir "/etc/hadoop/yarn-site.xml") (yarn-site-config test))
- (write-config! (str install-dir "/etc/hadoop/core-site.xml") (core-site-config test))
- (c/exec :echo (c/lit "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64") :>> (str install-dir "/etc/hadoop/hadoop-env.sh"))
+ (write-configuration! test)
(start-name-node-formatted! test node)
(start-data-node! test node)
(start-resource-manager! test node)
(start-node-manager! test node)))
- (teardown! [_ test node]
+ (teardown! [_ _ _]
(info "Teardown Hadoop")
(c/su
(cu/grepkill! "hadoop")
- (c/exec (c/lit (str "rm -rf /tmp/hadoop-* ||:")))))
+ (c/exec :rm :-rf install-dir)))
db/LogFiles
(log-files [_ _ _]
(c/su
- (concat (find-files! (str install-dir "/logs"))
- (if (cu/exists? yarn-log-dir)
- (do
- (c/exec :chmod :-R :777 yarn-log-dir)
- (find-files! yarn-log-dir))
- []))))))
+ (meh (c/exec :chmod :-R :755 log-dir))
+ (fu/find-files! log-dir)))))
diff --git a/flink-jepsen/src/jepsen/flink/utils.clj b/flink-jepsen/src/jepsen/flink/utils.clj
index 2c7328f21a8..1aa53efe7ae 100644
--- a/flink-jepsen/src/jepsen/flink/utils.clj
+++ b/flink-jepsen/src/jepsen/flink/utils.clj
@@ -51,6 +51,20 @@
(recur op (assoc keys :retries (dec retries))))
(success r)))))
+(defn find-files!
+ "Lists files recursively given a directory. If the directory does not exist, an empty collection
+ is returned."
+ [dir]
+ (let [files (try
+ (c/exec :find dir :-type :f)
+ (catch Exception e
+ (if (.contains (.getMessage e) "No such file or directory")
+ ""
+ (throw e))))]
+ (->>
+ (clojure.string/split files #"\n")
+ (remove clojure.string/blank?))))
+
;;; runit process supervisor (http://smarden.org/runit/)
(def runit-version "2.1.2-3")
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services