You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/09/14 21:12:43 UTC
hadoop git commit: HDDS-446. Provide shaded artifact to start ozone
service as a datanode plugin. Contributed by Elek Marton
Repository: hadoop
Updated Branches:
refs/heads/trunk b0a659c8c -> b95aa5606
HDDS-446. Provide shaded artifact to start ozone service as a datanode plugin. Contributed by Elek Marton
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b95aa560
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b95aa560
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b95aa560
Branch: refs/heads/trunk
Commit: b95aa560628b527b1534a7f148bd61605deb90b3
Parents: b0a659c
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Fri Sep 14 14:11:53 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Fri Sep 14 14:12:03 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 8 +++
hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 +++++
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 ++++++++++++++++
.../src/main/compose/ozone-hdfs/docker-config | 75 ++++++++++++++++++++
hadoop-ozone/objectstore-service/pom.xml | 52 ++++++++++++++
5 files changed, 212 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index d91c7ef..128ce10 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -143,8 +143,16 @@ run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-$
run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
+
+#shaded ozonefs
mkdir -p "./share/hadoop/ozonefs"
cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
+
+#shaded datanode service
+mkdir -p "./share/hadoop/ozoneplugin"
+cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar"
+
+
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-dist/src/main/compose/ozone-hdfs/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env b/hadoop-dist/src/main/compose/ozone-hdfs/.env
new file mode 100644
index 0000000..c437513
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HADOOP_VERSION=3.1.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
new file mode 100644
index 0000000..ed18e5c
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ namenode:
+ image: flokkr/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9870:9870
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
+ command: ["hdfs", "namenode"]
+ datanode:
+ image: flokkr/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9864
+ volumes:
+ - ../..:/opt/ozone
+ command: ["hdfs","datanode"]
+ environment:
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["ozone","om"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
new file mode 100644
index 0000000..ef54872
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=ozoneManager
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=true
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b95aa560/hadoop-ozone/objectstore-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml
index b79919f..b20f1e2 100644
--- a/hadoop-ozone/objectstore-service/pom.xml
+++ b/hadoop-ozone/objectstore-service/pom.xml
@@ -46,6 +46,25 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+
+ <dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<version>1.5.9</version>
@@ -62,6 +81,39 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependencies>
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>plugin</shadedClassifierName>
+ <artifactSet>
+ <includes>
+ <!-- artifacts with provided scope will be excluded anyway -->
+ <include>*:*:*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org