You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/09/27 19:06:47 UTC
hadoop git commit: HDFS-12469. Ozone: Create docker-compose
definition to easily test real clusters. Contributed by Elek, Marton.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 fec0e745e -> 056a97833
HDFS-12469. Ozone: Create docker-compose definition to easily test real clusters. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/056a9783
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/056a9783
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/056a9783
Branch: refs/heads/HDFS-7240
Commit: 056a9783337f7e384f651cf86b30abf995d1ead8
Parents: fec0e74
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 27 12:04:09 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 27 12:04:09 2017 -0700
----------------------------------------------------------------------
dev-support/compose/ozone/.env | 1 +
dev-support/compose/ozone/docker-compose.yaml | 40 +++++++++++++++++
dev-support/compose/ozone/docker-config | 17 ++++++++
.../src/site/markdown/OzoneGettingStarted.md.vm | 45 ++++++++++++++++++++
4 files changed, 103 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/056a9783/dev-support/compose/ozone/.env
----------------------------------------------------------------------
diff --git a/dev-support/compose/ozone/.env b/dev-support/compose/ozone/.env
new file mode 100644
index 0000000..948e2d5
--- /dev/null
+++ b/dev-support/compose/ozone/.env
@@ -0,0 +1 @@
+VERSION=3.1.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/056a9783/dev-support/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/dev-support/compose/ozone/docker-compose.yaml b/dev-support/compose/ozone/docker-compose.yaml
new file mode 100644
index 0000000..1b89310
--- /dev/null
+++ b/dev-support/compose/ozone/docker-compose.yaml
@@ -0,0 +1,40 @@
+version: "3"
+services:
+ namenode:
+ image: flokkr/hadoop-runner:latest
+ hostname: namenode
+ volumes:
+ - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+ ports:
+ - 50070:50070
+ - 9870:9870
+ environment:
+ ENSURE_NAMENODE_DIR: /data/namenode
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","namenode"]
+ datanode:
+ image: flokkr/hadoop-runner:latest
+ volumes:
+ - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/hdfs","datanode"]
+ ksm:
+ image: flokkr/hadoop-runner:latest
+ volumes:
+ - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+ ports:
+ - 9874:9874
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","ksm"]
+ scm:
+ image: flokkr/hadoop-runner:latest
+ volumes:
+ - ../../../hadoop-dist/target/hadoop-${VERSION}:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/056a9783/dev-support/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/dev-support/compose/ozone/docker-config b/dev-support/compose/ozone/docker-config
new file mode 100644
index 0000000..72e95a2
--- /dev/null
+++ b/dev-support/compose/ozone/docker-config
@@ -0,0 +1,17 @@
+CORE-SITE.XML_fs.defaultFS: "hdfs://namenode:9000"
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/056a9783/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
index b9abd84..ec9d65c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
@@ -41,9 +41,54 @@ example of the tarball that will be generated.
* `~/apache/hadoop/hadoop-dist/target/${project.version}.tar.gz`
+At this point we have an option to setup a physical cluster or run ozone via
+docker.
+
+Running Ozone via Docker
+------------------------
+
+This assumes that you have a running docker setup on the machine. Please run
+these following commands to see ozone in action.
+
+ Go to the directory where the docker compose files exist.
+
+
+ - `cd dev-support/compose/ozone`
+
+Tell docker to start ozone, this will start a KSM, SCM and a single datanode in
+the background.
+
+
+ - `docker-compose up -d`
+
+Now let us run some work load against ozone, to do that we will run corona.
+
+This will log into the datanode and run bash.
+
+ - `docker-compose exec datanode bash`
+ - `cd hadoop/bin`
+
+Now you can run the oz command shell or corona the ozone load generator.
+
+This is the command to run corona.
+
+ - `./hdfs corona -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
+
+You can checkout the KSM UI to see the requests information.
+
+ - `http://localhost:9874/`
+
+If you need more datanode you can scale up:
+
+ - `docker-compose scale datanode=3`
+
+Running Ozone using a real cluster
+----------------------------------
+
Please proceed to setup a hadoop cluster by creating the hdfs-site.xml and
other configuration files that are needed for your cluster.
+
### Ozone Configuration
Ozone relies on its own configuration file called `ozone-site.xml`. It is
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org