You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ga...@apache.org on 2019/04/10 11:48:03 UTC
[flink] branch master updated (ce59213 -> cc3c2d6)
This is an automated email from the ASF dual-hosted git repository.
gary pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from ce59213 [FLINK-11884][table] Ported projection validation on top of Expressions
new bdb30a7 [hotfix][tests] Fix tests for jepsen.flink.generator/inc-by-factor
new c3bd1bd [FLINK-12123][tests] Upgrade Jepsen to 0.1.13
new cc3c2d6 [hotfix][tests] Shut down HDFS and YARN cleanly.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-jepsen/docker/Dockerfile-control | 6 +--
flink-jepsen/docker/Dockerfile-db | 32 +++++++++++++---
flink-jepsen/project.clj | 2 +-
flink-jepsen/src/jepsen/flink/checker.clj | 43 ++++++++++++----------
flink-jepsen/src/jepsen/flink/flink.clj | 14 +++----
flink-jepsen/src/jepsen/flink/hadoop.clj | 21 ++++++++++-
flink-jepsen/src/jepsen/flink/mesos.clj | 35 ++++++++++++------
flink-jepsen/src/jepsen/flink/nemesis.clj | 17 +++++++--
flink-jepsen/src/jepsen/flink/utils.clj | 9 +++--
flink-jepsen/test/jepsen/flink/checker_test.clj | 5 +--
.../flink/{nemesis_test.clj => generator_test.clj} | 12 +++---
11 files changed, 131 insertions(+), 65 deletions(-)
rename flink-jepsen/test/jepsen/flink/{nemesis_test.clj => generator_test.clj} (74%)
[flink] 03/03: [hotfix][tests] Shut down HDFS and YARN cleanly.
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit cc3c2d671d4280d8a95d413f1d4fff353f33991c
Author: Gary Yao <ga...@apache.org>
AuthorDate: Wed Apr 10 09:47:04 2019 +0200
[hotfix][tests] Shut down HDFS and YARN cleanly.
Use provided scripts to shut down HDFS and YARN instead of grepkill! function.
Using grepkill! one can run situations where flink and hadoop processes are
killed concurrently, which results in an exception.
---
flink-jepsen/src/jepsen/flink/hadoop.clj | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
diff --git a/flink-jepsen/src/jepsen/flink/hadoop.clj b/flink-jepsen/src/jepsen/flink/hadoop.clj
index 4ba7859..2fbb445 100644
--- a/flink-jepsen/src/jepsen/flink/hadoop.clj
+++ b/flink-jepsen/src/jepsen/flink/hadoop.clj
@@ -115,18 +115,37 @@
(info "Start DataNode")
(c/exec (str install-dir "/sbin/hadoop-daemon.sh") :--config hadoop-conf-dir :--script :hdfs :start :datanode)))
+(defn stop-data-node!
+ []
+ (c/exec (str install-dir "/sbin/hadoop-daemon.sh") :--config hadoop-conf-dir :--script :hdfs :stop :datanode))
+
(defn start-resource-manager!
[test node]
(when (= node (resource-manager (:nodes test)))
(info "Start ResourceManager")
(c/exec (str install-dir "/sbin/yarn-daemon.sh") :--config hadoop-conf-dir :start :resourcemanager)))
+(defn stop-resource-manager!
+ []
+ (c/exec (str install-dir "/sbin/yarn-daemon.sh") :--config hadoop-conf-dir :stop :resourcemanager))
+
(defn start-node-manager!
[test node]
(when (some #{node} (data-nodes (:nodes test)))
(info "Start NodeManager")
(c/exec (str install-dir "/sbin/yarn-daemon.sh") :--config hadoop-conf-dir :start :nodemanager)))
+(defn stop-node-manager!
+ []
+ (c/exec (str install-dir "/sbin/yarn-daemon.sh") :--config hadoop-conf-dir :stop :nodemanager))
+
+(defn stop-all!
+ []
+ (stop-name-node!)
+ (stop-data-node!)
+ (stop-resource-manager!)
+ (stop-node-manager!))
+
(defn db
[url]
(reify db/DB
@@ -143,7 +162,7 @@
(teardown! [_ _ _]
(info "Teardown Hadoop")
(c/su
- (cu/grepkill! "hadoop")
+ (stop-all!)
(c/exec :rm :-rf install-dir)))
db/LogFiles
[flink] 01/03: [hotfix][tests] Fix tests for
jepsen.flink.generator/inc-by-factor
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit bdb30a73751c3c713ed89cffc220662b0f97bf56
Author: Gary Yao <ga...@apache.org>
AuthorDate: Tue Apr 9 09:35:42 2019 +0200
[hotfix][tests] Fix tests for jepsen.flink.generator/inc-by-factor
---
.../jepsen/flink/{nemesis_test.clj => generator_test.clj} | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/flink-jepsen/test/jepsen/flink/nemesis_test.clj b/flink-jepsen/test/jepsen/flink/generator_test.clj
similarity index 74%
rename from flink-jepsen/test/jepsen/flink/nemesis_test.clj
rename to flink-jepsen/test/jepsen/flink/generator_test.clj
index 488631f..54a2247 100644
--- a/flink-jepsen/test/jepsen/flink/nemesis_test.clj
+++ b/flink-jepsen/test/jepsen/flink/generator_test.clj
@@ -14,19 +14,19 @@
;; See the License for the specific language governing permissions and
;; limitations under the License.
-(ns jepsen.flink.nemesis-test
+(ns jepsen.flink.generator-test
(:require [clojure.test :refer :all])
- (:require [jepsen.flink.nemesis :refer :all]))
+ (:require [jepsen.flink.generator :refer :all]))
(deftest inc-by-factor-test
(testing "Should not increase if factor is 1."
- (is (= 10 (@#'jepsen.flink.nemesis/inc-by-factor 10 1))))
+ (is (= 10 (@#'jepsen.flink.generator/inc-by-factor 10 1))))
(testing "Should increase by factor."
- (is (= 15 (@#'jepsen.flink.nemesis/inc-by-factor 10 1.5))))
+ (is (= 15 (@#'jepsen.flink.generator/inc-by-factor 10 1.5))))
(testing "Should round down."
- (is (= 15 (@#'jepsen.flink.nemesis/inc-by-factor 10 1.52))))
+ (is (= 15 (@#'jepsen.flink.generator/inc-by-factor 10 1.52))))
(testing "Should throw if factor < 1."
- (is (thrown? AssertionError (@#'jepsen.flink.nemesis/inc-by-factor 1 0)))))
+ (is (thrown? AssertionError (@#'jepsen.flink.generator/inc-by-factor 1 0)))))
[flink] 02/03: [FLINK-12123][tests] Upgrade Jepsen to 0.1.13
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit c3bd1bd00961cd6165993f018ef59ae9379af482
Author: Gary Yao <ga...@apache.org>
AuthorDate: Tue Apr 9 12:01:24 2019 +0200
[FLINK-12123][tests] Upgrade Jepsen to 0.1.13
Upgrade Jepsen dependency to get support for Debian Stretch (Jessie has reached
EOL):
* Adapt for new checker interface
* Install Marathon manually because there are no packages for Debian Stretch
* Upgrade Marathon to 1.7 because I cannot find the 1.6.322 binaries anywhere
* Update Dockerfiles to Debian Stretch
This closes #8131.
---
flink-jepsen/docker/Dockerfile-control | 6 ++--
flink-jepsen/docker/Dockerfile-db | 32 +++++++++++++++---
flink-jepsen/project.clj | 2 +-
flink-jepsen/src/jepsen/flink/checker.clj | 43 +++++++++++++------------
flink-jepsen/src/jepsen/flink/flink.clj | 14 ++++----
flink-jepsen/src/jepsen/flink/mesos.clj | 35 +++++++++++++-------
flink-jepsen/src/jepsen/flink/nemesis.clj | 17 ++++++++--
flink-jepsen/src/jepsen/flink/utils.clj | 9 ++++--
flink-jepsen/test/jepsen/flink/checker_test.clj | 5 ++-
9 files changed, 105 insertions(+), 58 deletions(-)
diff --git a/flink-jepsen/docker/Dockerfile-control b/flink-jepsen/docker/Dockerfile-control
index 96198e3..e71aa57 100644
--- a/flink-jepsen/docker/Dockerfile-control
+++ b/flink-jepsen/docker/Dockerfile-control
@@ -16,11 +16,9 @@
# limitations under the License.
################################################################################
-FROM debian:jessie
+FROM debian:stretch
-RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list && \
- apt-get update && \
- apt-get install -y -t jessie-backports openjdk-8-jdk && \
+RUN apt-get update && \
apt-get install -qqy \
less \
libjna-java \
diff --git a/flink-jepsen/docker/Dockerfile-db b/flink-jepsen/docker/Dockerfile-db
index cb60efc..5bda9be 100644
--- a/flink-jepsen/docker/Dockerfile-db
+++ b/flink-jepsen/docker/Dockerfile-db
@@ -16,12 +16,34 @@
# limitations under the License.
################################################################################
-FROM debian:jessie
+FROM debian:stretch
-RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list && \
- apt-get update && \
- apt-get install -y -t jessie-backports openjdk-8-jdk && \
- apt-get install -y apt-utils bzip2 curl faketime iproute iptables iputils-ping less libzip2 logrotate man man-db net-tools ntpdate psmisc python rsyslog runit sudo sysvinit sysvinit-core sysvinit-utils tar unzip vim wget
+RUN apt-get update && \
+ apt-get install -qqy \
+ apt-utils \
+ bzip2 \
+ curl \
+ faketime \
+ gnupg \
+ iproute \
+ iptables \
+ iputils-ping \
+ less \
+ libzip4 \
+ logrotate \
+ man \
+ man-db \
+ net-tools \
+ ntpdate \
+ openjdk-8-jdk \
+ psmisc python \
+ rsyslog \
+ runit \
+ sudo \
+ tar \
+ unzip \
+ vim \
+ wget
RUN apt-get update && \
apt-get -y install openssh-server && \
diff --git a/flink-jepsen/project.clj b/flink-jepsen/project.clj
index bcf84c7..5fb7c37 100644
--- a/flink-jepsen/project.clj
+++ b/flink-jepsen/project.clj
@@ -22,7 +22,7 @@
:dependencies [[org.clojure/clojure "1.9.0"],
[cheshire "5.8.0"]
[clj-http "3.8.0"]
- [jepsen "0.1.11"],
+ [jepsen "0.1.13"],
[jepsen.zookeeper "0.1.0"]
[org.clojure/data.xml "0.0.8"]
[zookeeper-clj "0.9.4" :exclusions [org.slf4j/slf4j-log4j12]]]
diff --git a/flink-jepsen/src/jepsen/flink/checker.clj b/flink-jepsen/src/jepsen/flink/checker.clj
index d5ff6e0..f43bdd1 100644
--- a/flink-jepsen/src/jepsen/flink/checker.clj
+++ b/flink-jepsen/src/jepsen/flink/checker.clj
@@ -149,11 +149,9 @@
; ignore other client operations
this)))))
-(defn job-running-within-grace-period
+(defn- job-running-within-grace-period
([job-running-healthy-threshold job-recovery-grace-period job-cancellation-grace-period]
- (JobRunningWithinGracePeriod. {} 0 nil job-running-healthy-threshold job-recovery-grace-period job-cancellation-grace-period false nil))
- ([job-running-healthy-threshold job-recovery-grace-period]
- (job-running-within-grace-period job-running-healthy-threshold job-recovery-grace-period 10)))
+ (JobRunningWithinGracePeriod. {} 0 nil job-running-healthy-threshold job-recovery-grace-period job-cancellation-grace-period false nil)))
(defn- history->jobs-running?-value
[history]
@@ -216,19 +214,24 @@
history)))
(defn job-running-checker
- []
- (reify
- checker/Checker
- (check [_ test model history _]
- (let [job-ids (history->job-ids history)
- individual-job-histories (map (partial history->single-job-history history) job-ids)
- final-models (map (partial compute-final-model model) individual-job-histories)
- inconsistent-or-unhealthy (or (empty? job-ids)
- (some model/inconsistent? final-models)
- (some (complement healthy?) final-models))
- result-map (select-keys test [:nemesis-gen :deployment-mode])]
- (if inconsistent-or-unhealthy
- (into result-map {:valid? false
- :final-models final-models})
- (into result-map {:valid? true
- :final-models final-models}))))))
+ ([job-running-healthy-threshold job-recovery-grace-period]
+ (job-running-checker job-running-healthy-threshold job-recovery-grace-period 10))
+ ([job-running-healthy-threshold job-recovery-grace-period job-cancellation-grace-period]
+ (reify
+ checker/Checker
+ (check [_ test history _]
+ (let [job-ids (history->job-ids history)
+ individual-job-histories (map (partial history->single-job-history history) job-ids)
+ model (job-running-within-grace-period job-running-healthy-threshold
+ job-recovery-grace-period
+ job-cancellation-grace-period)
+ final-models (map (partial compute-final-model model) individual-job-histories)
+ inconsistent-or-unhealthy (or (empty? job-ids)
+ (some model/inconsistent? final-models)
+ (some (complement healthy?) final-models))
+ result-map (select-keys test [:nemesis-gen :deployment-mode])]
+ (if inconsistent-or-unhealthy
+ (into result-map {:valid? false
+ :final-models final-models})
+ (into result-map {:valid? true
+ :final-models final-models})))))))
diff --git a/flink-jepsen/src/jepsen/flink/flink.clj b/flink-jepsen/src/jepsen/flink/flink.clj
index a3698ab..5447c05 100644
--- a/flink-jepsen/src/jepsen/flink/flink.clj
+++ b/flink-jepsen/src/jepsen/flink/flink.clj
@@ -36,9 +36,9 @@
(def default-flink-dist-url "https://archive.apache.org/dist/flink/flink-1.6.0/flink-1.6.0-bin-hadoop28-scala_2.11.tgz")
(def hadoop-dist-url "https://archive.apache.org/dist/hadoop/common/hadoop-2.8.3/hadoop-2.8.3.tar.gz")
(def kafka-dist-url "http://mirror.funkfreundelandshut.de/apache/kafka/2.0.1/kafka_2.11-2.0.1.tgz")
-(def deb-zookeeper-package "3.4.9-3+deb8u1")
-(def deb-mesos-package "1.5.0-2.0.2")
-(def deb-marathon-package "1.6.322")
+(def deb-zookeeper-package "3.4.9-3+deb9u1")
+(def deb-mesos-package "1.5.0-2.0.1")
+(def marathon-dist-url "https://downloads.mesosphere.io/marathon/builds/1.7.189-48bfd6000/marathon-1.7.189-48bfd6000.tgz")
(def dbs
{:flink-yarn-job (fdb/yarn-job-db)
@@ -47,7 +47,7 @@
:flink-mesos-session (fdb/flink-mesos-app-master)
:hadoop (hadoop/db hadoop-dist-url)
:kafka (kafka/db kafka-dist-url)
- :mesos (mesos/db deb-mesos-package deb-marathon-package)
+ :mesos (mesos/db deb-mesos-package marathon-dist-url)
:zookeeper (zk/db deb-zookeeper-package)})
(def poll-jobs-running {:type :invoke, :f :jobs-running?, :value nil})
@@ -84,9 +84,6 @@
:os debian/os
:db (fdb/combined-db dbs)
:nemesis (fn/nemesis)
- :model (flink-checker/job-running-within-grace-period
- job-running-healthy-threshold
- job-recovery-grace-period)
:generator (let [stop (atom nil)]
(->> (fg/stoppable-generator stop (client-gen))
(gen/nemesis
@@ -95,7 +92,8 @@
job-running-healthy-threshold
job-recovery-grace-period))))
:client (create-client)
- :checker (flink-checker/job-running-checker)})
+ :checker (flink-checker/job-running-checker job-running-healthy-threshold
+ job-recovery-grace-period)})
(assoc opts :concurrency 1)))
(defn- keys->allowed-values-help-text
diff --git a/flink-jepsen/src/jepsen/flink/mesos.clj b/flink-jepsen/src/jepsen/flink/mesos.clj
index 400ed42..e944089 100644
--- a/flink-jepsen/src/jepsen/flink/mesos.clj
+++ b/flink-jepsen/src/jepsen/flink/mesos.clj
@@ -37,7 +37,8 @@
;;; Marathon
-(def marathon-bin "/usr/bin/marathon")
+(def marathon-install-dir "/opt/marathon")
+(def marathon-bin (str marathon-install-dir "/bin/marathon"))
(def zk-marathon-namespace "marathon")
(def marathon-rest-port 8080)
@@ -69,6 +70,7 @@
(str "--master=" (zookeeper-uri test zk-namespace))
(str "--recovery_timeout=30secs")
(str "--work_dir=" slave-dir)
+ (str "--no-systemd_enable_support")
(str "--resources='cpus:8'")]))
(defn create-mesos-master-supervised-service!
@@ -118,21 +120,24 @@
(c/lit (str log-dir "/*"))
(c/lit (str slave-dir "/*"))))))
-;;; Marathon functions
-
-(defn install!
- [test node mesos-version marathon-version]
+(defn install-mesos!
+ [mesos-version]
(c/su
(debian/add-repo! :mesosphere
- "deb http://repos.mesosphere.com/debian jessie main"
+ "deb http://repos.mesosphere.com/debian stretch main"
"keyserver.ubuntu.com"
"E56151BF")
- (debian/install {:mesos mesos-version
- :marathon marathon-version})
- (c/exec :mkdir :-p "/var/run/mesos")
+ (debian/install {:mesos mesos-version})
(c/exec :mkdir :-p master-dir)
(c/exec :mkdir :-p slave-dir)))
+;;; Marathon functions
+
+(defn install-marathon!
+ [marathon-dist-url]
+ (c/su
+ (cu/install-archive! marathon-dist-url marathon-install-dir)))
+
(defn marathon-cmd
"Returns the command to run the marathon."
[test node]
@@ -168,11 +173,19 @@
[test]
(str "http://" (first (sort (:nodes test))) ":" marathon-rest-port))
+;;; Mesos & Marathon DB
+
+(defn install!
+ [mesos-version marathon-dist-url]
+ (c/su
+ (install-mesos! mesos-version)
+ (install-marathon! marathon-dist-url)))
+
(defn db
- [mesos-version marathon-version]
+ [mesos-version marathon-dist-url]
(reify db/DB
(setup! [this test node]
- (install! test node mesos-version marathon-version)
+ (install! mesos-version marathon-dist-url)
(start-master! test node)
(start-slave! test node)
(start-marathon! test node))
diff --git a/flink-jepsen/src/jepsen/flink/nemesis.clj b/flink-jepsen/src/jepsen/flink/nemesis.clj
index 37e8421..6afa48d 100644
--- a/flink-jepsen/src/jepsen/flink/nemesis.clj
+++ b/flink-jepsen/src/jepsen/flink/nemesis.clj
@@ -25,12 +25,24 @@
[jepsen.flink.client :refer :all]
[jepsen.flink.generator :as fgen]
[jepsen.flink.hadoop :as fh]
- [jepsen.flink.zookeeper :refer :all]))
+ [jepsen.flink.zookeeper :refer :all]
+ [slingshot.slingshot :refer [try+]]))
(def job-submit-grace-period
"Period after job submission in which job managers must not fail."
60)
+(defn- grepkill!
+ [pattern]
+ (try+
+ (cu/grepkill! pattern)
+ ;; HACK:
+ ;; On Debian Stretch, Jepsen's grepkill! throws an exception if the pattern does not match any
+ ;; processes. We are swallowing the exception here because the process we are attempting to kill
+ ;; might not be (re-)started yet.
+ ;; For details, see https://github.com/jepsen-io/jepsen/issues/366
+ (catch [:type :jepsen.control/nonzero-exit :exit 123] _)))
+
(defn kill-processes
([pattern] (kill-processes rand-nth pattern))
([targeter pattern]
@@ -38,8 +50,7 @@
(setup! [this test] this)
(invoke! [this test op]
(let [nodes (-> test :nodes targeter ju/coll)]
- (c/on-many nodes
- (c/su (cu/grepkill! pattern)))
+ (c/on-many nodes (c/su (grepkill! pattern)))
(assoc op :value nodes)))
(teardown! [this test]))))
diff --git a/flink-jepsen/src/jepsen/flink/utils.clj b/flink-jepsen/src/jepsen/flink/utils.clj
index 7ccae12..5d16568 100644
--- a/flink-jepsen/src/jepsen/flink/utils.clj
+++ b/flink-jepsen/src/jepsen/flink/utils.clj
@@ -71,12 +71,14 @@
;;; runit process supervisor (http://smarden.org/runit/)
-(def runit-version "2.1.2-3")
+(def runit-version "2.1.2-9.2")
+(def runit-systemd-version "2.1.2-9.2")
(defn- install-process-supervisor!
"Installs the process supervisor."
[]
- (debian/install {:runit runit-version}))
+ (debian/install {:runit runit-version
+ :runit-systemd runit-systemd-version}))
(defn create-supervised-service!
"Registers a service with the process supervisor and starts it."
@@ -91,6 +93,7 @@
"exec 2>&1"
(str "exec " cmd)]) :> run-script)
(c/exec :chmod :+x run-script)
+ (c/exec :mkdir :-p "/etc/service")
(c/exec :ln :-sfT service-dir (str "/etc/service/" service-name)))))
(defn stop-supervised-service!
@@ -108,4 +111,4 @@
;; HACK:
;; Remove all symlinks in /etc/service except sshd.
;; This is only relevant when tests are run in Docker because there sshd is started using runit.
- (meh (c/exec :find (c/lit (str "/etc/service -maxdepth 1 -type l ! -name 'sshd' -delete"))))))
+ (meh (c/exec :find (c/lit (str "/etc/service -mindepth 1 -maxdepth 1 -type l -not -name 'sshd' -delete"))))))
diff --git a/flink-jepsen/test/jepsen/flink/checker_test.clj b/flink-jepsen/test/jepsen/flink/checker_test.clj
index d1ade16..6f4ca46 100644
--- a/flink-jepsen/test/jepsen/flink/checker_test.clj
+++ b/flink-jepsen/test/jepsen/flink/checker_test.clj
@@ -30,11 +30,10 @@
(is (= [false true false] (all-jobs-running?-history history)))))
(deftest job-running-checker-test
- (let [checker (job-running-checker)
+ (let [checker (job-running-checker 3 60 10)
test {}
- model (job-running-within-grace-period 3 60 10)
opts {}
- check (fn [history] (checker/check checker test model history opts))
+ check (fn [history] (checker/check checker test history opts))
job-running-value {"3886d6b547969c3f15c53896bb496b8f" true}
job-not-running-value {"3886d6b547969c3f15c53896bb496b8f" false}]
(testing "Model should be inconsistent if job is not running after grace period."