You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/07/09 15:27:34 UTC
[hadoop] branch ozone-0.4.1 updated: HDDS-1742. Merge ozone-perf
and ozonetrace example clusters
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/ozone-0.4.1 by this push:
new b7b00bd HDDS-1742. Merge ozone-perf and ozonetrace example clusters
b7b00bd is described below
commit b7b00bdda94b7855b7bc02368cec0bd53f69748c
Author: pifta <pi...@cloudera.com>
AuthorDate: Tue Jul 9 17:21:16 2019 +0200
HDDS-1742. Merge ozone-perf and ozonetrace example clusters
Closes #1057
(cherry picked from commit 1dc58c5b043f7a285cdaf58dc87d3f3509e9e0b0)
---
.../dist/src/main/compose/ozoneperf/README.md | 10 ++-
.../src/main/compose/ozoneperf/docker-compose.yaml | 15 ++++
.../dist/src/main/compose/ozoneperf/docker-config | 11 +++
hadoop-ozone/dist/src/main/compose/ozonetrace/.env | 19 -----
.../main/compose/ozonetrace/docker-compose.yaml | 64 -----------------
.../dist/src/main/compose/ozonetrace/docker-config | 84 ----------------------
6 files changed, 33 insertions(+), 170 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
index d24eed4..62328e0 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
@@ -15,7 +15,8 @@
# Compose files for local performance tests
This directory contains docker-compose definition for an ozone cluster where
-all the metrics are saved to a prometheus instance.
+all the metrics are saved to a prometheus instance, and profiling and Jaeger
+tracing is turned on and set up.
Prometheus follows a pull based approach where the metrics are published
on a HTTP endpoint.
@@ -36,8 +37,8 @@ Note: The freon test will be started after 30 seconds.
You can check the ozone web ui:
-OzoneManager: https://localhost:9874
-SCM: https://localhost:9876
+OzoneManager: http://localhost:9874
+SCM: http://localhost:9876
You can check the ozone metrics from the prometheus web ui.
@@ -50,3 +51,6 @@ http://localhost:3000
Default dashboards available are:
Ozone - Object Metrics
Ozone - RPC Metrics
+
+You can access the Jaeger UI at:
+http://localhost:16686
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
index 0dda92b..ddbb182 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
@@ -47,6 +47,12 @@ services:
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
command: ["ozone","scm"]
+ jaeger:
+ image: jaegertracing/all-in-one:latest
+ environment:
+ COLLECTOR_ZIPKIN_HTTP_PORT: 9411
+ ports:
+ - 16686:16686
prometheus:
image: prom/prometheus
volumes:
@@ -72,3 +78,12 @@ services:
command: ["-config","/etc/grafana/grafana.ini"]
ports:
- 3000:3000
+ s3g:
+ image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9878:9878
+ env_file:
+ - ./docker-config
+ command: ["ozone","s3g"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
index eb23aad..538376e 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
@@ -25,10 +25,17 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.prometheus.endpoint.enabled=true
+OZONE-SITE.XML_hdds.profiler.endpoint.enabled=true
+
+ASYNC_PROFILER_HOME=/opt/profiler
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+JAEGER_SAMPLER_PARAM=1
+JAEGER_SAMPLER_TYPE=const
+JAEGER_AGENT_HOST=jaeger
+
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -37,3 +44,7 @@ LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.grpc.client.GrpcClientProtocolClient=WARN
+LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log
+LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3
diff --git a/hadoop-ozone/dist/src/main/compose/ozonetrace/.env b/hadoop-ozone/dist/src/main/compose/ozonetrace/.env
deleted file mode 100644
index 8eabf58..0000000
--- a/hadoop-ozone/dist/src/main/compose/ozonetrace/.env
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
-HADOOP_VERSION=3
-HADOOP_RUNNER_VERSION=${docker.hadoop-runner.version}
\ No newline at end of file
diff --git a/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-compose.yaml
deleted file mode 100644
index d910d82..0000000
--- a/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-compose.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- jaeger:
- image: jaegertracing/all-in-one:latest
- environment:
- COLLECTOR_ZIPKIN_HTTP_PORT: 9411
- ports:
- - 16686:16686
- datanode:
- image: apache/hadoop-runner:${HADOOP_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["ozone","datanode"]
- env_file:
- - ./docker-config
- om:
- image: apache/hadoop-runner:${HADOOP_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
- env_file:
- - ./docker-config
- command: ["ozone","om"]
- scm:
- image: apache/hadoop-runner:${HADOOP_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["ozone","scm"]
- s3g:
- image: apache/hadoop-runner:${HADOOP_RUNNER_VERSION}
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9878:9878
- env_file:
- - ./docker-config
- command: ["ozone","s3g"]
diff --git a/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config b/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config
deleted file mode 100644
index 59747f7..0000000
--- a/hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
-
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-JAEGER_SAMPLER_PARAM=1
-JAEGER_SAMPLER_TYPE=const
-JAEGER_AGENT_HOST=jaeger
-
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log
-LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org