You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/09/24 17:10:44 UTC
[1/4] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
Repository: hadoop
Updated Branches:
refs/heads/trunk d060cbea4 -> 62f817d32
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/objectstore-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml
index b20f1e2..ce8fe4c 100644
--- a/hadoop-ozone/objectstore-service/pom.xml
+++ b/hadoop-ozone/objectstore-service/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Object Store REST Service</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index 3efb33b..a3f69f6 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Manager Server</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
@@ -63,28 +58,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>copy web resources</id>
- <phase>compile</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <copy toDir="${project.build.directory}/webapps">
- <fileset dir="${basedir}/src/main/webapps">
- <exclude name="**/proto-web.xml"/>
- </fileset>
- </copy>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
@@ -93,19 +66,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals>
<goal>unpack</goal>
</goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ <outputDirectory>${project.build.outputDirectory}
+ </outputDirectory>
+ <includes>webapps/static/**/*.*</includes>
+ </artifactItem>
+ </artifactItems>
+ <overWriteSnapshots>true</overWriteSnapshots>
+ </configuration>
</execution>
</executions>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-framework</artifactId>
- <outputDirectory>${project.build.directory}/</outputDirectory>
- <includes>webapps/static/**/*.*</includes>
- </artifactItem>
- </artifactItems>
- <overWriteSnapshots>true</overWriteSnapshots>
- </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
new file mode 100644
index 0000000..ba54cb2
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
@@ -0,0 +1,70 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <meta name="description" content="Ozone Manager">
+
+ <title>Ozone Manager</title>
+
+ <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
+ <link href="static/hadoop.css" rel="stylesheet">
+ <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
+
+ <link href="static/ozone.css" rel="stylesheet">
+
+</head>
+
+<body ng-app="ozoneManager">
+
+<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
+ aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="#">Ozone Manager</a>
+ </div>
+ <navmenu
+ metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
+ </div>
+</header>
+
+<div class="container-fluid">
+ <ng-view></ng-view>
+</div><!-- /.container -->
+
+<script src="static/jquery-3.3.1.min.js"></script>
+<script src="static/angular-1.6.4.min.js"></script>
+<script src="static/angular-route-1.6.4.min.js"></script>
+<script src="static/d3-3.5.17.min.js"></script>
+<script src="static/nvd3-1.8.5.min.js"></script>
+<script src="static/angular-nvd3-1.0.9.min.js"></script>
+<script src="static/ozone.js"></script>
+<script src="ozoneManager.js"></script>
+<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
new file mode 100644
index 0000000..e442adc
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+body {
+ padding-top: 50px;
+}
+.starter-template {
+ padding: 40px 15px;
+ text-align: center;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
new file mode 100644
index 0000000..0821899
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<overview>
+</overview>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
new file mode 100644
index 0000000..15fba2f
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
@@ -0,0 +1,44 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<h1>OzoneManager Metrics</h1>
+
+<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
+ <h2>{{type}}</h2>
+ <div class="container">
+ <div class="col-md-6">
+ <h3>Requests ({{numbers.ops}} ops)</h3>
+ <nvd3 options="$ctrl.graphOptions"
+ data="numbers.all"></nvd3>
+ </div>
+ <div class="col-md-6">
+ <h3>Failures</h3>
+ <nvd3 options="$ctrl.graphOptions"
+ data="numbers.failures"></nvd3>
+ </div>
+ </div>
+</div>
+
+<div ng-show="$ctrl.metrics.others.length > 0">
+ <h2>Other JMX properties</h2>
+
+ <table class="table">
+ <tr ng-repeat="metric in $ctrl.metrics.others">
+ <td>{{metric.key}}</td>
+ <td>{{metric.value}}</td>
+ </tr>
+ </table>
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
new file mode 100644
index 0000000..ca03554
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ "use strict";
+
+ var isIgnoredJmxKeys = function (key) {
+ return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
+ };
+
+ angular.module('ozoneManager', ['ozone', 'nvd3']);
+ angular.module('ozoneManager').config(function ($routeProvider) {
+ $routeProvider
+ .when("/metrics/ozoneManager", {
+ template: "<om-metrics></om-metrics>"
+ });
+ });
+ angular.module('ozoneManager').component('omMetrics', {
+ templateUrl: 'om-metrics.html',
+ controller: function ($http) {
+ var ctrl = this;
+
+ ctrl.graphOptions = {
+ chart: {
+ type: 'pieChart',
+ height: 500,
+ x: function (d) {
+ return d.key;
+ },
+ y: function (d) {
+ return d.value;
+ },
+ showLabels: true,
+ labelType: 'value',
+ duration: 500,
+ labelThreshold: 0.01,
+ valueFormat: function(d) {
+ return d3.format('d')(d);
+ },
+ legend: {
+ margin: {
+ top: 5,
+ right: 35,
+ bottom: 5,
+ left: 0
+ }
+ }
+ }
+ };
+
+
+ $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
+ .then(function (result) {
+
+ var groupedMetrics = {others: [], nums: {}};
+ var metrics = result.data.beans[0]
+ for (var key in metrics) {
+ var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
+ if (numericalStatistic) {
+ var type = numericalStatistic[1];
+ var name = numericalStatistic[2];
+ var failed = numericalStatistic[3];
+ groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
+ failures: [],
+ all: []
+ };
+ if (failed) {
+ groupedMetrics.nums[type].failures.push({
+ key: name,
+ value: metrics[key]
+ })
+ } else {
+ if (name == "Ops") {
+ groupedMetrics.nums[type].ops = metrics[key]
+ } else {
+ groupedMetrics.nums[type].all.push({
+ key: name,
+ value: metrics[key]
+ })
+ }
+ }
+ } else if (isIgnoredJmxKeys(key)) {
+ //ignore
+ } else {
+ groupedMetrics.others.push({
+ 'key': key,
+ 'value': metrics[key]
+ });
+ }
+ }
+ ctrl.metrics = groupedMetrics;
+ })
+ }
+ });
+
+})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
deleted file mode 100644
index ba54cb2..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
+++ /dev/null
@@ -1,70 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <meta name="description" content="Ozone Manager">
-
- <title>Ozone Manager</title>
-
- <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
- <link href="static/hadoop.css" rel="stylesheet">
- <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
- <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="ozoneManager">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
- aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="#">Ozone Manager</a>
- </div>
- <navmenu
- metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
- </div>
-</header>
-
-<div class="container-fluid">
- <ng-view></ng-view>
-</div><!-- /.container -->
-
-<script src="static/jquery-3.3.1.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="ozoneManager.js"></script>
-<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
deleted file mode 100644
index e442adc..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-body {
- padding-top: 50px;
-}
-.starter-template {
- padding: 40px 15px;
- text-align: center;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
deleted file mode 100644
index 0821899..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<overview>
-</overview>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
deleted file mode 100644
index 15fba2f..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h1>OzoneManager Metrics</h1>
-
-<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
- <h2>{{type}}</h2>
- <div class="container">
- <div class="col-md-6">
- <h3>Requests ({{numbers.ops}} ops)</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.all"></nvd3>
- </div>
- <div class="col-md-6">
- <h3>Failures</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.failures"></nvd3>
- </div>
- </div>
-</div>
-
-<div ng-show="$ctrl.metrics.others.length > 0">
- <h2>Other JMX properties</h2>
-
- <table class="table">
- <tr ng-repeat="metric in $ctrl.metrics.others">
- <td>{{metric.key}}</td>
- <td>{{metric.value}}</td>
- </tr>
- </table>
-</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
deleted file mode 100644
index ca03554..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- var isIgnoredJmxKeys = function (key) {
- return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
- };
-
- angular.module('ozoneManager', ['ozone', 'nvd3']);
- angular.module('ozoneManager').config(function ($routeProvider) {
- $routeProvider
- .when("/metrics/ozoneManager", {
- template: "<om-metrics></om-metrics>"
- });
- });
- angular.module('ozoneManager').component('omMetrics', {
- templateUrl: 'om-metrics.html',
- controller: function ($http) {
- var ctrl = this;
-
- ctrl.graphOptions = {
- chart: {
- type: 'pieChart',
- height: 500,
- x: function (d) {
- return d.key;
- },
- y: function (d) {
- return d.value;
- },
- showLabels: true,
- labelType: 'value',
- duration: 500,
- labelThreshold: 0.01,
- valueFormat: function(d) {
- return d3.format('d')(d);
- },
- legend: {
- margin: {
- top: 5,
- right: 35,
- bottom: 5,
- left: 0
- }
- }
- }
- };
-
-
- $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
- .then(function (result) {
-
- var groupedMetrics = {others: [], nums: {}};
- var metrics = result.data.beans[0]
- for (var key in metrics) {
- var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
- if (numericalStatistic) {
- var type = numericalStatistic[1];
- var name = numericalStatistic[2];
- var failed = numericalStatistic[3];
- groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
- failures: [],
- all: []
- };
- if (failed) {
- groupedMetrics.nums[type].failures.push({
- key: name,
- value: metrics[key]
- })
- } else {
- if (name == "Ops") {
- groupedMetrics.nums[type].ops = metrics[key]
- } else {
- groupedMetrics.nums[type].all.push({
- key: name,
- value: metrics[key]
- })
- }
- }
- } else if (isIgnoredJmxKeys(key)) {
- //ignore
- } else {
- groupedMetrics.others.push({
- 'key': key,
- 'value': metrics[key]
- });
- }
- }
- ctrl.metrics = groupedMetrics;
- })
- }
- });
-
-})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index c45aacd..8174b74 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -28,8 +28,6 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
</properties>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index c73be60..8d3e161 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -19,9 +19,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project-dist</artifactId>
+ <artifactId>hadoop-project</artifactId>
<version>3.2.0-SNAPSHOT</version>
- <relativePath>../hadoop-project-dist</relativePath>
+ <relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-ozone</artifactId>
<version>0.3.0-SNAPSHOT</version>
@@ -42,6 +42,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>ozonefs</module>
<module>integration-test</module>
<module>objectstore-service</module>
+ <module>datanode</module>
+ <module>dist</module>
<module>docs</module>
</modules>
@@ -161,6 +163,25 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<excludeFilterFile combine.self="override"></excludeFilterFile>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-classpath-descriptor</id>
+ <phase>package</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <attach>true</attach>
+ <prefix>$HDDS_LIB_JARS_DIR</prefix>
+ <outputFilterFile>true</outputFilterFile>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index eeec595..b04bbf1 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -28,21 +28,33 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Tools</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
@@ -59,6 +71,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>jmh-generator-annprocess</artifactId>
<version>1.19</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-integration-test</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
new file mode 100644
index 0000000..a2df50d
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Freon, with MiniOzoneCluster and validate data.
+ */
+public class TestDataValidate {
+
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(5).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void ratisTestLargeKey() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setKeySize(20971520);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
+ }
+
+ @Test
+ public void standaloneTestLargeKey() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setKeySize(20971520);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
+ }
+
+ @Test
+ public void validateWriteTest() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(2);
+ randomKeyGenerator.setNumOfBuckets(5);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertTrue(randomKeyGenerator.getValidateWrites());
+ Assert.assertNotEquals(0, randomKeyGenerator.getTotalKeysValidated());
+ Assert.assertNotEquals(0, randomKeyGenerator
+ .getSuccessfulValidationCount());
+ Assert.assertEquals(0, randomKeyGenerator
+ .getUnsuccessfulValidationCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
new file mode 100644
index 0000000..d21d399
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Freon, with MiniOzoneCluster.
+ */
+public class TestRandomKeyGenerator {
+
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void defaultTest() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(2);
+ randomKeyGenerator.setNumOfBuckets(5);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.call();
+ Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(10240 - 36, randomKeyGenerator.getKeyValueLength());
+ }
+
+ @Test
+ public void multiThread() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(10);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setNumOfThreads(10);
+ randomKeyGenerator.setKeySize(10240);
+ randomKeyGenerator.call();
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ }
+
+ @Test
+ public void ratisTest3() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(10);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setNumOfThreads(10);
+ randomKeyGenerator.setKeySize(10240);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.call();
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
new file mode 100644
index 0000000..a3ff6c8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.scm.cli.SQLCLI;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.KeyArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the CLI that transforms om.db into SQLite DB files.
+ */
+@RunWith(Parameterized.class)
+public class TestOmSQLCli {
+ private MiniOzoneCluster cluster = null;
+ private StorageHandler storageHandler;
+ private UserArgs userArgs;
+ private OzoneConfiguration conf;
+ private SQLCLI cli;
+
+ private String userName = "userTest";
+ private String adminName = "adminTest";
+ private String volumeName0 = "volumeTest0";
+ private String volumeName1 = "volumeTest1";
+ private String bucketName0 = "bucketTest0";
+ private String bucketName1 = "bucketTest1";
+ private String bucketName2 = "bucketTest2";
+ private String keyName0 = "key0";
+ private String keyName1 = "key1";
+ private String keyName2 = "key2";
+ private String keyName3 = "key3";
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ // Uncomment the below line if we support leveldb in future.
+ //{OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private String metaStoreType;
+
+ public TestOmSQLCli(String type) {
+ metaStoreType = type;
+ }
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster.waitForClusterToBeReady();
+ storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
+ userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ cluster.waitForClusterToBeReady();
+
+ VolumeArgs createVolumeArgs0 = new VolumeArgs(volumeName0, userArgs);
+ createVolumeArgs0.setUserName(userName);
+ createVolumeArgs0.setAdminName(adminName);
+ storageHandler.createVolume(createVolumeArgs0);
+ VolumeArgs createVolumeArgs1 = new VolumeArgs(volumeName1, userArgs);
+ createVolumeArgs1.setUserName(userName);
+ createVolumeArgs1.setAdminName(adminName);
+ storageHandler.createVolume(createVolumeArgs1);
+
+ BucketArgs bucketArgs0 = new BucketArgs(volumeName0, bucketName0, userArgs);
+ storageHandler.createBucket(bucketArgs0);
+ BucketArgs bucketArgs1 = new BucketArgs(volumeName1, bucketName1, userArgs);
+ storageHandler.createBucket(bucketArgs1);
+ BucketArgs bucketArgs2 = new BucketArgs(volumeName0, bucketName2, userArgs);
+ storageHandler.createBucket(bucketArgs2);
+
+ KeyArgs keyArgs0 =
+ new KeyArgs(volumeName0, bucketName0, keyName0, userArgs);
+ keyArgs0.setSize(100);
+ KeyArgs keyArgs1 =
+ new KeyArgs(volumeName1, bucketName1, keyName1, userArgs);
+ keyArgs1.setSize(200);
+ KeyArgs keyArgs2 =
+ new KeyArgs(volumeName0, bucketName2, keyName2, userArgs);
+ keyArgs2.setSize(300);
+ KeyArgs keyArgs3 =
+ new KeyArgs(volumeName0, bucketName2, keyName3, userArgs);
+ keyArgs3.setSize(400);
+
+ OutputStream stream = storageHandler.newKeyWriter(keyArgs0);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs1);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs2);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs3);
+ stream.close();
+
+ cluster.getOzoneManager().stop();
+ cluster.getStorageContainerManager().stop();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
+ cli = new SQLCLI(conf);
+ }
+
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ // After HDDS-357, we have to fix SQLCli.
+ // TODO: fix SQLCli
+ @Ignore
+ @Test
+ public void testOmDB() throws Exception {
+ String dbOutPath = GenericTestUtils.getTempPath(
+ UUID.randomUUID() + "/out_sql.db");
+
+ String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbPath = dbRootPath + "/" + OM_DB_NAME;
+ String[] args = {"-p", dbPath, "-o", dbOutPath};
+
+ cli.run(args);
+
+ Connection conn = connectDB(dbOutPath);
+ String sql = "SELECT * FROM volumeList";
+ ResultSet rs = executeQuery(conn, sql);
+ List<String> expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String userNameRs = rs.getString("userName");
+ String volumeNameRs = rs.getString("volumeName");
+ assertEquals(userName, userNameRs.substring(1));
+ assertTrue(expectedValues.remove(volumeNameRs));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM volumeInfo";
+ rs = executeQuery(conn, sql);
+ expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String adName = rs.getString("adminName");
+ String ownerName = rs.getString("ownerName");
+ String volumeName = rs.getString("volumeName");
+ assertEquals(adminName, adName);
+ assertEquals(userName, ownerName);
+ assertTrue(expectedValues.remove(volumeName));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM aclInfo";
+ rs = executeQuery(conn, sql);
+ expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String adName = rs.getString("adminName");
+ String ownerName = rs.getString("ownerName");
+ String volumeName = rs.getString("volumeName");
+ String type = rs.getString("type");
+ String uName = rs.getString("userName");
+ String rights = rs.getString("rights");
+ assertEquals(adminName, adName);
+ assertEquals(userName, ownerName);
+ assertEquals("USER", type);
+ assertEquals(userName, uName);
+ assertEquals("READ_WRITE", rights);
+ assertTrue(expectedValues.remove(volumeName));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM bucketInfo";
+ rs = executeQuery(conn, sql);
+ HashMap<String, String> expectedMap = new HashMap<>();
+ expectedMap.put(bucketName0, volumeName0);
+ expectedMap.put(bucketName2, volumeName0);
+ expectedMap.put(bucketName1, volumeName1);
+ while (rs.next()) {
+ String volumeName = rs.getString("volumeName");
+ String bucketName = rs.getString("bucketName");
+ boolean versionEnabled = rs.getBoolean("versionEnabled");
+ String storegeType = rs.getString("storageType");
+ assertEquals(volumeName, expectedMap.remove(bucketName));
+ assertFalse(versionEnabled);
+ assertEquals("DISK", storegeType);
+ }
+ assertEquals(0, expectedMap.size());
+
+ sql = "SELECT * FROM keyInfo";
+ rs = executeQuery(conn, sql);
+ HashMap<String, List<String>> expectedMap2 = new HashMap<>();
+ // no data written, data size will be 0
+ expectedMap2.put(keyName0,
+ Arrays.asList(volumeName0, bucketName0, "0"));
+ expectedMap2.put(keyName1,
+ Arrays.asList(volumeName1, bucketName1, "0"));
+ expectedMap2.put(keyName2,
+ Arrays.asList(volumeName0, bucketName2, "0"));
+ expectedMap2.put(keyName3,
+ Arrays.asList(volumeName0, bucketName2, "0"));
+ while (rs.next()) {
+ String volumeName = rs.getString("volumeName");
+ String bucketName = rs.getString("bucketName");
+ String keyName = rs.getString("keyName");
+ int dataSize = rs.getInt("dataSize");
+ List<String> vals = expectedMap2.remove(keyName);
+ assertNotNull(vals);
+ assertEquals(vals.get(0), volumeName);
+ assertEquals(vals.get(1), bucketName);
+ assertEquals(vals.get(2), Integer.toString(dataSize));
+ }
+ assertEquals(0, expectedMap2.size());
+
+ conn.close();
+ Files.delete(Paths.get(dbOutPath));
+ }
+
+ private ResultSet executeQuery(Connection conn, String sql)
+ throws SQLException {
+ Statement stmt = conn.createStatement();
+ return stmt.executeQuery(sql);
+ }
+
+ private Connection connectDB(String dbPath) throws Exception {
+ Class.forName("org.sqlite.JDBC");
+ String connectPath =
+ String.format("jdbc:sqlite:%s", dbPath);
+ return DriverManager.getConnection(connectPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
new file mode 100644
index 0000000..4026348
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright containerOwnership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
+import org.apache.hadoop.hdds.scm.container.ContainerMapping;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
+import org.apache.hadoop.ozone.scm.cli.SQLCLI;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
+import static org.apache.hadoop.ozone.OzoneConsts.KB;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class tests the CLI that transforms container into SQLite DB files.
+ */
+@RunWith(Parameterized.class)
+public class TestContainerSQLCli {
+
+ private EventQueue eventQueue;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private static String metaStoreType;
+
+ public TestContainerSQLCli(String type) {
+ metaStoreType = type;
+ }
+
+ private static SQLCLI cli;
+
+ private MiniOzoneCluster cluster;
+ private OzoneConfiguration conf;
+ private String datanodeIpAddress;
+
+ private ContainerMapping mapping;
+ private NodeManager nodeManager;
+ private BlockManagerImpl blockManager;
+
+ private HashMap<Long, Long> blockContainerMap;
+
+ private final static long DEFAULT_BLOCK_SIZE = 4 * KB;
+ private static HddsProtos.ReplicationFactor factor;
+ private static HddsProtos.ReplicationType type;
+ private static final String CONTAINER_OWNER = "OZONE";
+
+
+ @Before
+ public void setup() throws Exception {
+ blockContainerMap = new HashMap<>();
+
+ conf = new OzoneConfiguration();
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 2);
+ conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ if(conf.getBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
+ ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT)){
+ factor = HddsProtos.ReplicationFactor.THREE;
+ type = HddsProtos.ReplicationType.RATIS;
+ } else {
+ factor = HddsProtos.ReplicationFactor.ONE;
+ type = HddsProtos.ReplicationType.STAND_ALONE;
+ }
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2).build();
+ cluster.waitForClusterToBeReady();
+ datanodeIpAddress = cluster.getHddsDatanodes().get(0)
+ .getDatanodeDetails().getIpAddress();
+ cluster.getOzoneManager().stop();
+ cluster.getStorageContainerManager().stop();
+ eventQueue = new EventQueue();
+ nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
+ mapping = new ContainerMapping(conf, nodeManager, 128,
+ eventQueue);
+ blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
+ eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
+ GenericTestUtils.waitFor(() -> {
+ return !blockManager.isScmInChillMode();
+ }, 10, 1000 * 15);
+ // blockManager.allocateBlock() will create containers if there is none
+ // stored in levelDB. The number of containers to create is the value of
+ // OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2.
+ // so the first allocateBlock() will create two containers. A random one
+ // is assigned for the block.
+
+ // loop until both the two datanodes are up, try up to about 4 seconds.
+ for (int c = 0; c < 40; c++) {
+ if (nodeManager.getAllNodes().size() == 2) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ assertEquals(2, nodeManager.getAllNodes().size());
+ AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type,
+ factor, CONTAINER_OWNER);
+ blockContainerMap.put(ab1.getBlockID().getLocalID(),
+ ab1.getBlockID().getContainerID());
+
+ AllocatedBlock ab2;
+ // we want the two blocks on the two provisioned containers respectively,
+ // however blockManager picks containers randomly, keep retry until we
+ // assign the second block to the other container. This seems to be the only
+ // way to get the two containers.
+ // although each retry will create a block and assign to a container. So
+ // the size of blockContainerMap will vary each time the test is run.
+ while (true) {
+ ab2 = blockManager
+ .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER);
+ blockContainerMap.put(ab2.getBlockID().getLocalID(),
+ ab2.getBlockID().getContainerID());
+ if (ab1.getBlockID().getContainerID() !=
+ ab2.getBlockID().getContainerID()) {
+ break;
+ }
+ }
+
+ blockManager.close();
+ mapping.close();
+ nodeManager.close();
+
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
+ cli = new SQLCLI(conf);
+
+ }
+
+ @After
+ public void shutdown() throws InterruptedException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testConvertContainerDB() throws Exception {
+ String dbOutPath = GenericTestUtils.getTempPath(
+ UUID.randomUUID() + "/out_sql.db");
+ // TODO : the following will fail due to empty Datanode list, need to fix.
+ //String dnUUID = cluster.getDataNodes().get(0).getUuid();
+ String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB;
+ String[] args = {"-p", dbPath, "-o", dbOutPath};
+ Connection conn;
+ String sql;
+ ResultSet rs;
+
+ cli.run(args);
+
+ //verify the sqlite db
+ // only checks the container names are as expected. Because other fields
+ // such as datanode UUID are generated randomly each time
+ conn = connectDB(dbOutPath);
+ sql = "SELECT * FROM containerInfo";
+ rs = executeQuery(conn, sql);
+ ArrayList<Long> containerIDs = new ArrayList<>();
+ while (rs.next()) {
+ containerIDs.add(rs.getLong("containerID"));
+ //assertEquals(dnUUID, rs.getString("leaderUUID"));
+ }
+ /* TODO: fix this later when the SQLCLI is fixed.
+ assertTrue(containerIDs.size() == 2 &&
+ containerIDs.contains(pipeline1.getContainerName()) &&
+ containerIDs.contains(pipeline2.getContainerName()));
+
+ sql = "SELECT * FROM containerMembers";
+ rs = executeQuery(conn, sql);
+ containerIDs = new ArrayList<>();
+ while (rs.next()) {
+ containerIDs.add(rs.getLong("containerID"));
+ //assertEquals(dnUUID, rs.getString("datanodeUUID"));
+ }
+ assertTrue(containerIDs.size() == 2 &&
+ containerIDs.contains(pipeline1.getContainerName()) &&
+ containerIDs.contains(pipeline2.getContainerName()));
+
+ sql = "SELECT * FROM datanodeInfo";
+ rs = executeQuery(conn, sql);
+ int count = 0;
+ while (rs.next()) {
+ assertEquals(datanodeIpAddress, rs.getString("ipAddress"));
+ //assertEquals(dnUUID, rs.getString("datanodeUUID"));
+ count += 1;
+ }
+ // the two containers maybe on the same datanode, maybe not.
+ int expected = pipeline1.getLeader().getUuid().equals(
+ pipeline2.getLeader().getUuid())? 1 : 2;
+ assertEquals(expected, count);
+ */
+ Files.delete(Paths.get(dbOutPath));
+ }
+
+ private ResultSet executeQuery(Connection conn, String sql)
+ throws SQLException {
+ Statement stmt = conn.createStatement();
+ return stmt.executeQuery(sql);
+ }
+
+ private Connection connectDB(String dbPath) throws Exception {
+ Class.forName("org.sqlite.JDBC");
+ String connectPath =
+ String.format("jdbc:sqlite:%s", dbPath);
+ return DriverManager.getConnection(connectPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 794fdcc..8d08703 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -618,6 +618,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-datanode</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-integration-test</artifactId>
<version>${hdds.version}</version>
<type>test-jar</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/4] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
Posted by bh...@apache.org.
HDDS-447. Separate ozone-dist and hadoop-dist projects with real classpath separation. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62f817d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62f817d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62f817d3
Branch: refs/heads/trunk
Commit: 62f817d32e6304b4cc1f9b909e7b815316b3a021
Parents: d060cbe
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Mon Sep 24 10:10:11 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Mon Sep 24 10:10:11 2018 -0700
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 181 ------------
dev-support/bin/ozone-dist-tar-stitching | 43 ---
hadoop-dist/pom.xml | 118 --------
hadoop-dist/src/main/compose/README.md | 51 ----
hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 --
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 ----
.../src/main/compose/ozone-hdfs/docker-config | 76 -----
hadoop-dist/src/main/compose/ozone/.env | 17 --
.../src/main/compose/ozone/docker-compose.yaml | 49 ----
.../src/main/compose/ozone/docker-config | 75 -----
.../main/compose/ozonefs/docker-compose.yaml | 59 ----
.../src/main/compose/ozonefs/docker-config | 35 ---
hadoop-dist/src/main/compose/ozoneperf/.env | 17 --
.../src/main/compose/ozoneperf/README.md | 73 -----
.../src/main/compose/ozoneperf/compose-all.sh | 18 --
.../compose/ozoneperf/docker-compose-freon.yaml | 26 --
.../main/compose/ozoneperf/docker-compose.yaml | 64 -----
.../src/main/compose/ozoneperf/docker-config | 36 ---
hadoop-dist/src/main/compose/ozoneperf/init.sh | 21 --
.../src/main/compose/ozoneperf/prometheus.yml | 24 --
.../compose/ozonescripts/.ssh/authorized_keys | 16 --
.../src/main/compose/ozonescripts/.ssh/config | 18 --
.../main/compose/ozonescripts/.ssh/environment | 16 --
.../src/main/compose/ozonescripts/.ssh/id_rsa | 42 ---
.../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 --
.../src/main/compose/ozonescripts/Dockerfile | 33 ---
.../src/main/compose/ozonescripts/README.md | 38 ---
.../compose/ozonescripts/docker-compose.yaml | 42 ---
.../src/main/compose/ozonescripts/docker-config | 38 ---
hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 --
.../src/main/compose/ozonescripts/start.sh | 24 --
.../src/main/compose/ozonescripts/stop.sh | 17 --
hadoop-dist/src/main/ozone/README.txt | 51 ----
hadoop-dist/src/main/smoketest/README.md | 30 --
.../src/main/smoketest/basic/basic.robot | 47 ---
.../src/main/smoketest/basic/ozone-shell.robot | 82 ------
hadoop-dist/src/main/smoketest/commonlib.robot | 24 --
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 ---
hadoop-dist/src/main/smoketest/test.sh | 101 -------
hadoop-hdds/client/pom.xml | 5 -
hadoop-hdds/common/pom.xml | 2 -
hadoop-hdds/container-service/pom.xml | 5 -
hadoop-hdds/framework/pom.xml | 5 -
hadoop-hdds/pom.xml | 30 ++
hadoop-hdds/server-scm/pom.xml | 50 +---
.../src/main/resources/webapps/scm/index.html | 76 +++++
.../src/main/resources/webapps/scm/main.html | 20 ++
.../resources/webapps/scm/scm-overview.html | 60 ++++
.../src/main/resources/webapps/scm/scm.js | 54 ++++
.../server-scm/src/main/webapps/scm/index.html | 76 -----
.../server-scm/src/main/webapps/scm/main.html | 20 --
.../src/main/webapps/scm/scm-overview.html | 60 ----
.../server-scm/src/main/webapps/scm/scm.js | 54 ----
hadoop-hdds/tools/pom.xml | 8 -
hadoop-ozone/client/pom.xml | 6 -
hadoop-ozone/common/pom.xml | 5 -
hadoop-ozone/common/src/main/bin/ozone | 50 +++-
.../src/main/shellprofile.d/hadoop-ozone.sh | 24 --
hadoop-ozone/datanode/pom.xml | 49 ++++
.../dist/dev-support/bin/dist-layout-stitching | 113 ++++++++
.../dist/dev-support/bin/dist-tar-stitching | 45 +++
hadoop-ozone/dist/pom.xml | 184 ++++++++++++
hadoop-ozone/dist/src/main/compose/README.md | 51 ++++
.../dist/src/main/compose/ozone-hdfs/.env | 17 ++
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 ++++
.../src/main/compose/ozone-hdfs/docker-config | 76 +++++
hadoop-ozone/dist/src/main/compose/ozone/.env | 17 ++
.../src/main/compose/ozone/docker-compose.yaml | 49 ++++
.../dist/src/main/compose/ozone/docker-config | 75 +++++
.../main/compose/ozonefs/docker-compose.yaml | 59 ++++
.../dist/src/main/compose/ozonefs/docker-config | 35 +++
.../dist/src/main/compose/ozoneperf/.env | 17 ++
.../dist/src/main/compose/ozoneperf/README.md | 73 +++++
.../src/main/compose/ozoneperf/compose-all.sh | 18 ++
.../compose/ozoneperf/docker-compose-freon.yaml | 26 ++
.../main/compose/ozoneperf/docker-compose.yaml | 64 +++++
.../src/main/compose/ozoneperf/docker-config | 35 +++
.../dist/src/main/compose/ozoneperf/init.sh | 21 ++
.../src/main/compose/ozoneperf/prometheus.yml | 24 ++
.../compose/ozonescripts/.ssh/authorized_keys | 16 ++
.../src/main/compose/ozonescripts/.ssh/config | 18 ++
.../main/compose/ozonescripts/.ssh/environment | 16 ++
.../src/main/compose/ozonescripts/.ssh/id_rsa | 42 +++
.../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 ++
.../src/main/compose/ozonescripts/Dockerfile | 33 +++
.../src/main/compose/ozonescripts/README.md | 38 +++
.../compose/ozonescripts/docker-compose.yaml | 42 +++
.../src/main/compose/ozonescripts/docker-config | 38 +++
.../dist/src/main/compose/ozonescripts/ps.sh | 17 ++
.../dist/src/main/compose/ozonescripts/start.sh | 24 ++
.../dist/src/main/compose/ozonescripts/stop.sh | 17 ++
hadoop-ozone/dist/src/main/ozone/README.txt | 51 ++++
hadoop-ozone/dist/src/main/smoketest/README.md | 30 ++
.../dist/src/main/smoketest/basic/basic.robot | 47 +++
.../src/main/smoketest/basic/ozone-shell.robot | 82 ++++++
.../dist/src/main/smoketest/commonlib.robot | 24 ++
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 +++
hadoop-ozone/dist/src/main/smoketest/test.sh | 101 +++++++
hadoop-ozone/docs/pom.xml | 5 -
hadoop-ozone/integration-test/pom.xml | 14 -
.../hadoop/ozone/freon/TestDataValidate.java | 115 --------
.../ozone/freon/TestRandomKeyGenerator.java | 106 -------
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 -------------------
.../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ----------------
hadoop-ozone/objectstore-service/pom.xml | 5 -
hadoop-ozone/ozone-manager/pom.xml | 50 +---
.../resources/webapps/ozoneManager/index.html | 70 +++++
.../resources/webapps/ozoneManager/main.css | 23 ++
.../resources/webapps/ozoneManager/main.html | 18 ++
.../webapps/ozoneManager/om-metrics.html | 44 +++
.../webapps/ozoneManager/ozoneManager.js | 110 +++++++
.../src/main/webapps/ozoneManager/index.html | 70 -----
.../src/main/webapps/ozoneManager/main.css | 23 --
.../src/main/webapps/ozoneManager/main.html | 18 --
.../main/webapps/ozoneManager/om-metrics.html | 44 ---
.../main/webapps/ozoneManager/ozoneManager.js | 110 -------
hadoop-ozone/ozonefs/pom.xml | 2 -
hadoop-ozone/pom.xml | 25 +-
hadoop-ozone/tools/pom.xml | 32 ++-
.../hadoop/ozone/freon/TestDataValidate.java | 115 ++++++++
.../ozone/freon/TestRandomKeyGenerator.java | 106 +++++++
.../apache/hadoop/ozone/om/TestOmSQLCli.java | 285 +++++++++++++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 253 ++++++++++++++++
hadoop-project/pom.xml | 5 +
124 files changed, 3183 insertions(+), 3155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
deleted file mode 100755
index 00854b4..0000000
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# project.version
-VERSION=$1
-
-# project.build.directory
-BASEDIR=$2
-
-#hdds.version
-HDDS_VERSION=$3
-
-function run()
-{
- declare res
-
- echo "\$ ${*}"
- "${@}"
- res=$?
- if [[ ${res} != 0 ]]; then
- echo
- echo "Failed!"
- echo
- exit "${res}"
- fi
-}
-
-function findfileindir()
-{
- declare file="$1"
- declare dir="${2:-./share}"
- declare count
-
- count=$(find "${dir}" -iname "${file}" | wc -l)
-
- #shellcheck disable=SC2086
- echo ${count}
-}
-
-function copyifnotexists()
-{
- declare src="$1"
- declare dest="$2"
-
- declare srcname
- declare destdir
-
- declare child
- declare childpath
-
- if [[ -f "${src}" ]]; then
- srcname=${src##*/}
- if [[ "${srcname}" != *.jar ||
- $(findfileindir "${srcname}") -eq "0" ]]; then
- destdir=$(dirname "${dest}")
- mkdir -p "${destdir}"
- cp -p "${src}" "${dest}"
- fi
- else
- for childpath in "${src}"/*; do
- child="${childpath##*/}"
- if [[ "${child}" == "doc" ||
- "${child}" == "webapps" ]]; then
- mkdir -p "${dest}/${child}"
- cp -r "${src}/${child}"/* "${dest}/${child}"
- continue;
- fi
- copyifnotexists "${src}/${child}" "${dest}/${child}"
- done
- fi
-}
-
-#Copy all contents as is except the lib.
-#for libs check for existence in share directory, if not exist then only copy.
-function copy()
-{
- declare src="$1"
- declare dest="$2"
-
- declare child
- declare childpath
-
- if [[ -d "${src}" ]]; then
- for childpath in "${src}"/*; do
- child="${childpath##*/}"
-
- if [[ "${child}" == "share" ]]; then
- copyifnotexists "${src}/${child}" "${dest}/${child}"
- else
- if [[ -d "${src}/${child}" ]]; then
- mkdir -p "${dest}/${child}"
- cp -pr "${src}/${child}"/* "${dest}/${child}"
- else
- cp -pr "${src}/${child}" "${dest}/${child}"
- fi
- fi
- done
- fi
-}
-
-# shellcheck disable=SC2164
-ROOT=$(cd "${BASEDIR}"/../..;pwd)
-echo
-echo "Current directory $(pwd)"
-echo
-run rm -rf "ozone-${HDDS_VERSION}"
-run mkdir "ozone-${HDDS_VERSION}"
-run cd "ozone-${HDDS_VERSION}"
-run cp -p "${ROOT}/LICENSE.txt" .
-run cp -p "${ROOT}/NOTICE.txt" .
-
-# Copy hadoop-common first so that it have always have all dependencies.
-# Remaining projects will copy only libraries which are not present already in 'share' directory.
-run copy "${ROOT}/hadoop-common-project/hadoop-common/target/hadoop-common-${VERSION}" .
-
-
-# HDDS
-run copy "${ROOT}/hadoop-hdds/common/target/hadoop-hdds-common-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/framework/target/hadoop-hdds-server-framework-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/server-scm/target/hadoop-hdds-server-scm-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/container-service/target/hadoop-hdds-container-service-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/client/target/hadoop-hdds-client-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-hdds/tools/target/hadoop-hdds-tools-${HDDS_VERSION}" .
-
-# Ozone
-run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/client/target/hadoop-ozone-client-${HDDS_VERSION}" .
-run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}" .
-
-#shaded ozonefs
-mkdir -p "./share/hadoop/ozonefs"
-cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
-
-#shaded datanode service
-mkdir -p "./share/hadoop/ozoneplugin"
-cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
-
-
-# Optional documentation, could be missing
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
-cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./
-
-
-rm sbin/*all.sh
-rm sbin/*all.cmd
-
-#remove test and java sources
-find . -name "*tests.jar" | xargs rm
-find . -name "*sources.jar" | xargs rm
-find . -name jdiff -type d | xargs rm -rf
-
-#add ozone specific readme
-
-run cp "${ROOT}/hadoop-dist/src/main/ozone/README.txt" README.txt
-#Copy docker compose files and robot tests
-run cp -p -r "${ROOT}/hadoop-dist/src/main/compose" .
-run cp -p -r "${ROOT}/hadoop-dist/src/main/smoketest" .
-
-mkdir -p ./share/hadoop/mapreduce
-mkdir -p ./share/hadoop/yarn
-mkdir -p ./share/hadoop/hdfs
-echo
-echo "Hadoop Ozone dist layout available at: ${BASEDIR}/ozone-${HDDS_VERSION}"
-echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/dev-support/bin/ozone-dist-tar-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-tar-stitching b/dev-support/bin/ozone-dist-tar-stitching
deleted file mode 100755
index 93d0525..0000000
--- a/dev-support/bin/ozone-dist-tar-stitching
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# project.version
-VERSION=$1
-
-# project.build.directory
-BASEDIR=$2
-
-function run()
-{
- declare res
-
- echo "\$ ${*}"
- "${@}"
- res=$?
- if [[ ${res} != 0 ]]; then
- echo
- echo "Failed!"
- echo
- exit "${res}"
- fi
-}
-
-run tar -c -f "ozone-${VERSION}.tar" "ozone-${VERSION}"
-run gzip -f "ozone-${VERSION}.tar"
-echo
-echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz"
-echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index da05015..5447b6a 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -176,7 +176,6 @@
</argument>
<argument>${project.version}</argument>
<argument>${project.build.directory}</argument>
- <argument>${hdds.version}</argument>
</arguments>
</configuration>
</execution>
@@ -224,123 +223,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>hdds</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-ozone-manager</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-scm</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-tools</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-container-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-objectstore-service</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-tools</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-docs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-filesystem</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-dockerfile</id>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <phase>prepare-package</phase>
- <configuration>
- <outputDirectory>${project.build.directory}</outputDirectory>
- <resources>
- <resource>
- <directory>src/main/docker</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>dist-ozone</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${shell-executable}</executable>
- <workingDirectory>${project.build.directory}
- </workingDirectory>
- <arguments>
- <argument>
- ${basedir}/../dev-support/bin/ozone-dist-layout-stitching
- </argument>
- <argument>${project.version}</argument>
- <argument>${project.build.directory}</argument>
- <argument>${hdds.version}</argument>
- </arguments>
- </configuration>
- </execution>
- <execution>
- <id>tar-ozone</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${shell-executable}</executable>
- <workingDirectory>${project.build.directory}
- </workingDirectory>
- <arguments>
- <argument>${basedir}/../dev-support/bin/ozone-dist-tar-stitching
- </argument>
- <argument>${hdds.version}</argument>
- <argument>${project.build.directory}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md
deleted file mode 100644
index 8189d2c..0000000
--- a/hadoop-dist/src/main/compose/README.md
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Docker cluster definitions
-
-This directory contains multiple docker cluster definitions to start local pseudo cluster with different configuration.
-
-It helps to start local (multi-node like) pseudo cluster with docker and docker-compose and obviously it's not for production.
-
-You may find more information in the specific subdirectories but in generic you can use the following commands:
-
-## Usage
-
-To start a cluster go to a subdirectory and start the cluster:
-
-```
-docker-compose up -d
-```
-
-You can check the logs of all the components with:
-
-```
-docker-compose logs
-```
-
-In case of a problem you can destroy the cluster an delete all the local state with:
-
-```
-docker-compose down
-```
-
-(Note: a simple docker-compose stop may not delete all the local data).
-
-You can scale up and down the components:
-
-```
-docker-compose scale datanode=5
-```
-
-Usually the key webui ports are published on the docker host.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/.env b/hadoop-dist/src/main/compose/ozone-hdfs/.env
deleted file mode 100644
index c437513..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HADOOP_VERSION=3.1.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
deleted file mode 100644
index b89052d..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- namenode:
- image: flokkr/hadoop:${HADOOP_VERSION}
- ports:
- - 9870:9870
- env_file:
- - ./docker-config
- environment:
- ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
- command: ["hdfs", "namenode"]
- datanode:
- image: flokkr/hadoop:${HADOOP_VERSION}
- ports:
- - 9864
- volumes:
- - ../..:/opt/ozone
- command: ["hdfs","datanode"]
- environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
deleted file mode 100644
index 3b2819f..0000000
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-config
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/.env b/hadoop-dist/src/main/compose/ozone/.env
deleted file mode 100644
index 67eed25..0000000
--- a/hadoop-dist/src/main/compose/ozone/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
deleted file mode 100644
index 0a6a9d8..0000000
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
deleted file mode 100644
index f2c8db1..0000000
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
deleted file mode 100644
index a1e8748..0000000
--- a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
+++ /dev/null
@@ -1,59 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- hostname: ozoneManager
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
- hadooplast:
- image: flokkr/hadoop:3.1.0
- volumes:
- - ../..:/opt/ozone
- env_file:
- - ./docker-config
- environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
- command: ["watch","-n","100000","ls"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-config b/hadoop-dist/src/main/compose/ozonefs/docker-config
deleted file mode 100644
index 3171f08..0000000
--- a/hadoop-dist/src/main/compose/ozonefs/docker-config
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-dist/src/main/compose/ozoneperf/.env
deleted file mode 100644
index cac418a..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md
deleted file mode 100644
index 527ff41..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/README.md
+++ /dev/null
@@ -1,73 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Compose files for local performance tests
-
-This directory contains docker-compose definition for an ozone cluster where
-all the metrics are saved to a prometheus instance.
-
- Prometheus follows a pull based approach where the metrics are published
- on a HTTP endpoint.
-
- Our current approach:
-
- 1. A Java agent activates a prometheus metrics endpoint in every JVM instance
- (use `init.sh` to download the agent)
-
- 2. The Java agent publishes all the jmx parameters in prometheus format AND
- register the endpoint address to the consul.
-
- 3. Prometheus polls all the endpoints which are registered to consul.
-
-
-
-## How to use
-
-First of all download the required Java agent with running `./init.sh`
-
-After that you can start the cluster with docker-compose:
-
-```
-docker-compose up -d
-```
-
-After a while the cluster will be started. You can check the ozone web ui-s:
-
-https://localhost:9874
-https://localhost:9876
-
-You can also scale up the datanodes:
-
-```
-docker-compose scale datanode=3
-```
-
-Freon (Ozone test generator tool) is not part of docker-compose by default,
-you can activate it using `compose-all.sh` instead of `docker-compose`:
-
-```
-compose-all.sh up -d
-```
-
-Now Freon is running. Let's try to check the metrics from the local Prometheus:
-
-http://localhost:9090/graph
-
-Example queries:
-
-```
-Hadoop_OzoneManager_NumKeyCommits
-rate(Hadoop_OzoneManager_NumKeyCommits[10m])
-rate(Hadoop_Ozone_BYTES_WRITTEN[10m])
-```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
deleted file mode 100755
index 82ab8b3..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
deleted file mode 100644
index 60bdc4a..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- freon:
- image: apache/hadoop-runner
- volumes:
- - ../../ozone:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","freon"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
deleted file mode 100644
index 3b5cc7b..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: apache/hadoop-runner
- volumes:
- - ../..:/opt/hadoop
- - ./jmxpromo.jar:/opt/jmxpromo.jar
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
- consul:
- image: consul
- command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"]
- ports:
- - 8500:8500
- prometheus:
- image: prom/prometheus
- volumes:
- - "./prometheus.yml:/etc/prometheus.yml"
- command: ["--config.file","/etc/prometheus.yml"]
- ports:
- - 9090:9090
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config
deleted file mode 100644
index 454601e..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/docker-config
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=ozoneManager
-OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/init.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-dist/src/main/compose/ozoneperf/init.sh
deleted file mode 100755
index cf25398..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/init.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-EXPORTER_FILE="$DIR/jmxpromo.jar"
-if [ ! -f "$EXPORTER_FILE" ]; then
- wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE
-fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
deleted file mode 100644
index 80aa520..0000000
--- a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-global:
- scrape_interval: 15s # By default, scrape targets every 15 seconds.
-
-scrape_configs:
- - job_name: jmxexporter
- consul_sd_configs:
- - server: consul:8500
- services:
- - jmxexporter
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
deleted file mode 100644
index ae39052..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/authorized_keys
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
deleted file mode 100644
index 6506916..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/config
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-Host *
- UserKnownHostsFile /dev/null
- StrictHostKeyChecking no
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
deleted file mode 100644
index 5685453..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/environment
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
deleted file mode 100644
index 6632ce5..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
------BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEA4BJi6WJuAa1ratShvYYWVwmYBqxE57btHjU6NtVN1SnPZx/f
-6LezOpQGsLBXE/bl7uG+fD05Z378B/0wE5QhYwvJ9Ge0jsfhVOi90p/FEYfR2l+C
-9LRfLvO6AgA+HQa1BoYQd+norh/XQQVpcukSn32Cb642rW3OxsQTv3uTklDLapPw
-0lSpLJgYWcbrayorwjYw4rIfVYhiH/G+ckvzJRBpmlSgKYpNXyeYjpZaDPxoKQet
-GFf4W6nwnnG1oHojNYCvdoHTnAP1sdQu34/AUiOYg0hOnmXhfNIDN+NtZb8jw1sl
-YRWhm5GuK1wV/32/UEkQQPUU6oUSgBJgCH+PRwIDAQABAoIBAQDI1TH6ZNKchkck
-9XgSWsBjOqIcOQN5fCeDT8nho8WjLVpL3/Hcr+ngsxRcAXHK3xyvw33r9SQic1qJ
-/pC8u6RBFivo95qJ7vU0GXcp9TG4yLd6tui1U4WMm784U+dYNM7EDh1snSaECt3v
-1V3yNJ0QfnoOh2NShn0zAkOA+M4H8Nx2SudMCsjcbK9+fYxzW3hX+sJpMKdjG1HW
-DUz+I7cW7t0EGaVrgVSV+eR58LiXu+14YDNMrySiejB4nD2sKrx93XgiCBECCsBN
-GLQGJCztaXoAY+5Kf/aJ9EEf2wBF3GecRk+XIAd87PeDmeahLQAVkAJ/rD1vsKFs
-8kWi6CrhAoGBAP7leG/dMBhlfvROpBddIfWm2i8B+oZiAlSjdYGz+/ZhUaByXk18
-pupMGiiMQR1ziPnEg0gNgR2ZkH54qrXPn5WcQa4rlSEtUsZEp5v5WblhfX2QwKzY
-G/uhA+mB7wXpQkSmXo0LclfPF2teROQrG1OyfWkWbxFH4i3+em7sL95jAoGBAOEK
-v+wscqkMLW7Q8ONbWMCCBlmMHr6baB3VDCYZx25lr+GIF5zmJJFTmF2rq2VSAlts
-qx1AGmaUSo78kC5FuJvSNTL6a1Us5ucdthQZM3N8pAz+OAE+QEU+BsdA27yAh3tO
-yKDsMFNHKtXcgy5LeB5gzENLlNyw2jgkRv2Ef77NAoGAVH8DHqoHEH9Mx3XuRWR1
-JnaqKx0PzE5fEWmiQV3Fr5XxNivTgQJKXq7dmQVtbHLpPErdbhwz6fkHAjXD+UMb
-VsAWscL2y6m3n8wQd87/5EkiDWbXyDRXimGE53pQHviFJDa2bzEVNXCMBeaZFb4I
-cAViN1zdcrAOlUqfkXewIpsCgYB8wsXl/DpRB+RENGfn0+OfTjaQ/IKq72NIbq1+
-jfondQ6N/TICFQEe5HZrL9okoNOXteYjoD9CsWGoZdLVJGgVUvOVYImSvgMBDFK+
-T75bfzU/0sxfvBBLkviVDJsFpUf3D5VgybB86s6Po+HCD6r3RHjZshRESXOhflMx
-B3z+3QKBgE2Lwo0DuwUGkm6k8psyn3x8EiXNsiNw12cojicFTyKUYLHxMBeVbCLW
-3j3pxSggJgRuBLLzixUHbHp91r2ARTy28naK7R/la8yKVqK6ojcikN2mQsCHYtwB
-nuFwXr42ytn6G+9Wn4xT64tGjRCqyZn0/v0XsPjVCyrZ6G7EtNHP
------END RSA PRIVATE KEY-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub b/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
deleted file mode 100644
index ae39052..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
deleted file mode 100644
index 3bdcb0c..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM apache/hadoop-runner
-RUN sudo apt-get update && sudo apt-get install -y openssh-server
-
-RUN sudo mkdir -p /run/sshd
-RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
-RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config
-RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-#/opt/hadoop is mounted, we can't use it as a home
-RUN sudo usermod -d /opt hadoop
-ADD .ssh /opt/.ssh
-RUN sudo chown -R hadoop /opt/.ssh
-RUN sudo chown hadoop /opt
-RUN sudo chmod 600 /opt/.ssh/*
-RUN sudo chmod 700 /opt/.ssh
-
-RUN sudo sh -c 'echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/" >> /etc/profile'
-CMD ["sudo","/usr/sbin/sshd","-D"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/README.md b/hadoop-dist/src/main/compose/ozonescripts/README.md
deleted file mode 100644
index 2531fa4..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# start-ozone environment
-
-This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
-
-There are ssh connections between the containers and the start/stop scripts could handle the start/stop process
-similar to a real cluster.
-
-To use it, first start the cluster:
-
-```
-docker-copmose up -d
-```
-
-After a successfull startup (which starts only the ssh daemons) you can start ozone:
-
-```
-./start.sh
-```
-
-Check it the java processes are started:
-
-```
-./ps.sh
-```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
deleted file mode 100644
index 62f1163..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/docker-compose.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- datanode:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9864
- env_file:
- - ./docker-config
- om:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9874:9874
- env_file:
- - ./docker-config
- scm:
- build: .
- volumes:
- - ../..:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/docker-config b/hadoop-dist/src/main/compose/ozonescripts/docker-config
deleted file mode 100644
index 1afec73..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/docker-config
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
-CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
-OZONE-SITE.XML_ozone.ksm.address=ksm
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=true
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/ps.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/ps.sh b/hadoop-dist/src/main/compose/ozonescripts/ps.sh
deleted file mode 100755
index d5e2c38..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/ps.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/start.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/start.sh b/hadoop-dist/src/main/compose/ozonescripts/start.sh
deleted file mode 100755
index 3358b07..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/start.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1 docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
-docker-compose exec scm /opt/hadoop/bin/ozone scm -init
-docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
-#We need a running SCM for om objectstore creation
-#TODO create a utility to wait for the startup
-sleep 10
-docker-compose exec om /opt/hadoop/bin/ozone om -createObjectStore
-docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/compose/ozonescripts/stop.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonescripts/stop.sh b/hadoop-dist/src/main/compose/ozonescripts/stop.sh
deleted file mode 100755
index a3ce08a..0000000
--- a/hadoop-dist/src/main/compose/ozonescripts/stop.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-docker-compose exec scm /opt/hadoop/sbin/stop-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/ozone/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/ozone/README.txt b/hadoop-dist/src/main/ozone/README.txt
deleted file mode 100644
index 6bbd83f..0000000
--- a/hadoop-dist/src/main/ozone/README.txt
+++ /dev/null
@@ -1,51 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-This is the distribution of Apache Hadoop Ozone.
-
-Ozone is a submodule of Hadoop with separated release cycle. For more information, check
-
- http://ozone.hadoop.apache.org
-
- and
-
- https://cwiki.apache.org/confluence/display/HADOOP/Ozone+Contributor+Guide
-
-For more information about Hadoop, check:
-
- http://hadoop.apache.org
-
-This distribution includes cryptographic software. The country in
-which you currently reside may have restrictions on the import,
-possession, use, and/or re-export to another country, of
-encryption software. BEFORE using any encryption software, please
-check your country's laws, regulations and policies concerning the
-import, possession, or use, and re-export of encryption software, to
-see if this is permitted. See <http://www.wassenaar.org/> for more
-information.
-
-The U.S. Government Department of Commerce, Bureau of Industry and
-Security (BIS), has classified this software as Export Commodity
-Control Number (ECCN) 5D002.C.1, which includes information security
-software using or performing cryptographic functions with asymmetric
-algorithms. The form and manner of this Apache Software Foundation
-distribution makes it eligible for export under the License Exception
-ENC Technology Software Unrestricted (TSU) exception (see the BIS
-Export Administration Regulations, Section 740.13) for both object
-code and source code.
-
-The following provides more details on the included cryptographic
-software:
- Hadoop Core uses the SSL libraries from the Jetty project written
-by mortbay.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/README.md
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/README.md b/hadoop-dist/src/main/smoketest/README.md
deleted file mode 100644
index c521a54..0000000
--- a/hadoop-dist/src/main/smoketest/README.md
+++ /dev/null
@@ -1,30 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-## Ozone Acceptance Tests
-
-This directory contains a [robotframework](http://robotframework.org/) based test suite for Ozone to make it easier to check the current state of the package.
-
-You can run in in any environment after [installing](https://github.com/robotframework/robotframework/blob/master/INSTALL.rst)
-
-```
-cd $DIRECTORY_OF_OZONE
-robot smoketest/basic
-```
-
-The argument of the `robot` could be any robot file or directory.
-
-The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
-
-The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/basic/basic.robot
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/basic/basic.robot b/hadoop-dist/src/main/smoketest/basic/basic.robot
deleted file mode 100644
index a69450d..0000000
--- a/hadoop-dist/src/main/smoketest/basic/basic.robot
+++ /dev/null
@@ -1,47 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Smoketest ozone cluster startup
-Library OperatingSystem
-Resource ../commonlib.robot
-
-*** Variables ***
-${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${DATANODE_HOST} localhost
-
-
-*** Test Cases ***
-
-Test rest interface
- ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
- Should contain ${result} 201 Created
- ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
- Should contain ${result} 201 Created
- ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
- Should contain ${result} 200 OK
- ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
- Should contain ${result} 200 OK
-
-Check webui static resources
- ${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
- Should contain ${result} 200
- ${result} = Execute curl -s -I http://ozoneManager:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
- Should contain ${result} 200
-
-Start freon testing
- ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
- Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
- Should Not Contain ${result} ERROR
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/4] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
Posted by bh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot
deleted file mode 100644
index 14a5761..0000000
--- a/hadoop-dist/src/main/smoketest/basic/ozone-shell.robot
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Test ozone shell CLI usage
-Library OperatingSystem
-Resource ../commonlib.robot
-Test Timeout 2 minute
-
-*** Variables ***
-
-*** Test Cases ***
-RestClient without http port
- Test ozone shell http:// ozoneManager restwoport
-
-RestClient with http port
- Test ozone shell http:// ozoneManager:9874 restwport
-
-RestClient without host name
- Test ozone shell http:// ${EMPTY} restwohost
-
-RpcClient with port
- Test ozone shell o3:// ozoneManager:9862 rpcwoport
-
-RpcClient without host
- Test ozone shell o3:// ${EMPTY} rpcwport
-
-RpcClient without scheme
- Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme
-
-
-*** Keywords ***
-Test ozone shell
- [arguments] ${protocol} ${server} ${volume}
- ${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --user bilbo --quota 100TB --root
- Should not contain ${result} Failed
- Should contain ${result} Creating Volume: ${volume}
- ${result} = Execute ozone sh volume list ${protocol}${server}/ --user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
- Should contain ${result} createdOn
- ${result} = Execute ozone sh volume list --user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] | select(.volumeName=="${volume}")'
- Should contain ${result} createdOn
- Execute ozone sh volume update ${protocol}${server}/${volume} --user bill --quota 10TB
- ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
- Should Be Equal ${result} bill
- ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
- Should Be Equal ${result} 10
- Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1
- ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .storageType'
- Should Be Equal ${result} DISK
- ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --addAcl user:frodo:rw,group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="samwise") | .type'
- Should Be Equal ${result} GROUP
- ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
- Should Be Equal ${result} USER
- ${result} = Execute ozone sh bucket list ${protocol}${server}/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
- Should Be Equal ${result} ${volume}
- Run Keyword Test key handling ${protocol} ${server} ${volume}
- Execute ozone sh bucket delete ${protocol}${server}/${volume}/bb1
- Execute ozone sh volume delete ${protocol}${server}/${volume} --user bilbo
-
-Test key handling
- [arguments] ${protocol} ${server} ${volume}
- Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
- Execute rm -f NOTICE.txt.1
- Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt.1
- Execute ls -l NOTICE.txt.1
- ${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
- Should contain ${result} createdOn
- ${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
- Should Be Equal ${result} key1
- Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/commonlib.robot
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/commonlib.robot b/hadoop-dist/src/main/smoketest/commonlib.robot
deleted file mode 100644
index e2620fa..0000000
--- a/hadoop-dist/src/main/smoketest/commonlib.robot
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Keywords ***
-
-
-Execute
- [arguments] ${command}
- ${rc} ${output} = Run And Return Rc And Output ${command}
- Log ${output}
- Should Be Equal As Integers ${rc} 0
- [return] ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot
deleted file mode 100644
index fb7b98c..0000000
--- a/hadoop-dist/src/main/smoketest/ozonefs/ozonefs.robot
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Ozonefs test
-Library OperatingSystem
-Resource ../commonlib.robot
-
-*** Variables ***
-
-
-*** Test Cases ***
-Create volume and bucket
- Execute ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root
- Execute ozone sh bucket create http://ozoneManager/fstest/bucket1
-
-Check volume from ozonefs
- ${result} = Execute ozone fs -ls o3://bucket1.fstest/
-
-Create directory from ozonefs
- Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
- ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
- Should contain ${result} testdir/deep
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/smoketest/test.sh b/hadoop-dist/src/main/smoketest/test.sh
deleted file mode 100755
index 534bbb7..0000000
--- a/hadoop-dist/src/main/smoketest/test.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-execute_tests(){
- COMPOSE_FILE=$DIR/../compose/$1/docker-compose.yaml
- TESTS=$2
- echo "Executing test ${TESTS[*]} with $COMPOSE_FILE"
- docker-compose -f "$COMPOSE_FILE" down
- docker-compose -f "$COMPOSE_FILE" up -d
- docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get update
- docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get install -y python-pip
- docker-compose -f "$COMPOSE_FILE" exec datanode sudo pip install robotframework
- for TEST in "${TESTS[@]}"; do
- set +e
- docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot "smoketest/$TEST"
- set -e
- done
- if [ "$KEEP_RUNNING" = false ]; then
- docker-compose -f "$COMPOSE_FILE" down
- fi
-}
-RUN_ALL=true
-KEEP_RUNNING=false
-POSITIONAL=()
-while [[ $# -gt 0 ]]
-do
-key="$1"
-
-case $key in
- --env)
- DOCKERENV="$2"
- RUN_ALL=false
- shift # past argument
- shift # past value
- ;;
- --keep)
- KEEP_RUNNING=true
- shift # past argument
- ;;
- --help|-h|-help)
- cat << EOF
-
- Acceptance test executor for ozone.
-
- This is a lightweight test executor for ozone.
-
- You can run it with
-
- ./test.sh
-
- Which executes all the tests in all the available environments.
-
- Or you can run manually one test with
-
- ./test.sh --keep --env ozone-hdfs basic
-
- --keep means that docker cluster won't be stopped after the test (optional)
- --env defines the subdirectory under the compose dir
- The remaining parameters define the test suites under smoketest dir.
- Could be any directory or robot file relative to the smoketest dir.
-EOF
- exit 0
- ;;
- *)
- POSITIONAL+=("$1") # save it in an array for later
- shift # past argument
- ;;
-esac
-done
-
-if [ "$RUN_ALL" = true ]; then
-#
-# This is the definition of the ozone acceptance test suite
-#
-# We select the test suites and execute them on multiple type of clusters
-#
- DEFAULT_TESTS=("basic")
- execute_tests ozone "${DEFAULT_TESTS[@]}"
- TESTS=("ozonefs")
- execute_tests ozonefs "${TESTS[@]}"
-
-else
- execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
-fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml
index 6a4cc9d..53d72bb 100644
--- a/hadoop-hdds/client/pom.xml
+++ b/hadoop-hdds/client/pom.xml
@@ -29,11 +29,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop HDDS Client</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index d08a5a9..64ebe9b 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -30,8 +30,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<hdds.version>0.3.0-SNAPSHOT</hdds.version>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
<log4j2.version>2.11.0</log4j2.version>
<disruptor.version>3.4.2</disruptor.version>
<declared.hdds.version>${hdds.version}</declared.hdds.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/container-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml
index 3d4e581..54c5fad 100644
--- a/hadoop-hdds/container-service/pom.xml
+++ b/hadoop-hdds/container-service/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop HDDS Container Service</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/framework/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index cb380e2..511f321 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop HDDS Server Framework</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index ce4f234..59dfa79 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -119,6 +119,36 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<excludeFilterFile combine.self="override"></excludeFilterFile>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-classpath-descriptor</id>
+ <phase>package</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <attach>true</attach>
+ <prefix>$HDDS_LIB_JARS_DIR</prefix>
+ <outputFilterFile>true</outputFilterFile>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml
index 52f42e1..f34e848 100644
--- a/hadoop-hdds/server-scm/pom.xml
+++ b/hadoop-hdds/server-scm/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop HDDS SCM Server</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -106,28 +101,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>copy web resources</id>
- <phase>compile</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <copy toDir="${project.build.directory}/webapps">
- <fileset dir="${basedir}/src/main/webapps">
- <exclude name="**/proto-web.xml"/>
- </fileset>
- </copy>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
@@ -136,19 +109,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals>
<goal>unpack</goal>
</goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ <outputDirectory>${project.build.outputDirectory}
+ </outputDirectory>
+ <includes>webapps/static/**/*.*</includes>
+ </artifactItem>
+ </artifactItems>
+ <overWriteSnapshots>true</overWriteSnapshots>
+ </configuration>
</execution>
</executions>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-framework</artifactId>
- <outputDirectory>${project.build.directory}/</outputDirectory>
- <includes>webapps/static/**/*.*</includes>
- </artifactItem>
- </artifactItems>
- <overWriteSnapshots>true</overWriteSnapshots>
- </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html
new file mode 100644
index 0000000..2c943b6
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/index.html
@@ -0,0 +1,76 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <meta name="description" content="HDFS Storage Container Manager">
+
+ <title>HDFS Storage Container Manager</title>
+
+ <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
+ <link href="static/hadoop.css" rel="stylesheet">
+ <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
+
+ <link href="static/ozone.css" rel="stylesheet">
+
+</head>
+
+<body ng-app="scm">
+
+<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
+ aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="#">HDFS SCM</a>
+ </div>
+
+
+ <navmenu
+ metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
+
+
+ </div>
+</header>
+
+<div class="container-fluid" style="margin: 12pt">
+
+ <ng-view></ng-view>
+
+</div><!-- /.container -->
+
+<script src="static/jquery-3.3.1.min.js"></script>
+<script src="static/angular-1.6.4.min.js"></script>
+<script src="static/angular-route-1.6.4.min.js"></script>
+<script src="static/d3-3.5.17.min.js"></script>
+<script src="static/nvd3-1.8.5.min.js"></script>
+<script src="static/angular-nvd3-1.0.9.min.js"></script>
+<script src="static/ozone.js"></script>
+<script src="scm.js"></script>
+<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html
new file mode 100644
index 0000000..2666f81
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/main.html
@@ -0,0 +1,20 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<overview>
+ <scm-overview>
+ </scm-overview>
+</overview>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
new file mode 100644
index 0000000..fca23ba
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html
@@ -0,0 +1,60 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<h2>Node counts</h2>
+
+<table class="table table-bordered table-striped" class="col-md-6">
+ <tbody>
+ <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder">
+ <td>{{typestat.key}}</td>
+ <td>{{typestat.value}}</td>
+ </tr>
+ </tbody>
+</table>
+
+<h2>Status</h2>
+<table class="table table-bordered table-striped" class="col-md-6">
+ <tbody>
+ <tr>
+ <td>Client Rpc port</td>
+ <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td>
+ </tr>
+ <tr>
+ <td>Datanode Rpc port</td>
+ <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td>
+ </tr>
+ <tr>
+ <td>Block Manager: Open containers</td>
+ <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td>
+ </tr>
+ <tr>
+ <td>Node Manager: Minimum chill mode nodes</td>
+ <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td>
+ </tr>
+ <tr>
+ <td>Node Manager: Out-of-node chill mode</td>
+ <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td>
+ </tr>
+ <tr>
+ <td>Node Manager: Chill mode status</td>
+ <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td>
+ </tr>
+ <tr>
+ <td>Node Manager: Manual chill mode</td>
+ <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td>
+ </tr>
+ </tbody>
+</table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
new file mode 100644
index 0000000..bcfa8b7
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ "use strict";
+ angular.module('scm', ['ozone', 'nvd3']);
+
+ angular.module('scm').component('scmOverview', {
+ templateUrl: 'scm-overview.html',
+ require: {
+ overview: "^overview"
+ },
+ controller: function ($http) {
+ var ctrl = this;
+ $http.get("jmx?qry=Hadoop:service=BlockManager,name=*")
+ .then(function (result) {
+ ctrl.blockmanagermetrics = result.data.beans[0];
+ });
+ $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo")
+ .then(function (result) {
+ ctrl.nodemanagermetrics = result.data.beans[0];
+ });
+
+ var statusSortOrder = {
+ "HEALTHY": "a",
+ "STALE": "b",
+ "DEAD": "c",
+ "UNKNOWN": "z",
+ "DECOMMISSIONING": "x",
+ "DECOMMISSIONED": "y"
+ };
+ ctrl.nodeOrder = function (v1, v2) {
+ //status with non defined sort order will be "undefined"
+ return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value])
+ }
+
+ }
+ });
+
+})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/webapps/scm/index.html
deleted file mode 100644
index 2c943b6..0000000
--- a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html
+++ /dev/null
@@ -1,76 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <meta name="description" content="HDFS Storage Container Manager">
-
- <title>HDFS Storage Container Manager</title>
-
- <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
- <link href="static/hadoop.css" rel="stylesheet">
- <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
- <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="scm">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
- aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="#">HDFS SCM</a>
- </div>
-
-
- <navmenu
- metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
-
-
- </div>
-</header>
-
-<div class="container-fluid" style="margin: 12pt">
-
- <ng-view></ng-view>
-
-</div><!-- /.container -->
-
-<script src="static/jquery-3.3.1.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="scm.js"></script>
-<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/main.html b/hadoop-hdds/server-scm/src/main/webapps/scm/main.html
deleted file mode 100644
index 2666f81..0000000
--- a/hadoop-hdds/server-scm/src/main/webapps/scm/main.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<overview>
- <scm-overview>
- </scm-overview>
-</overview>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html b/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html
deleted file mode 100644
index fca23ba..0000000
--- a/hadoop-hdds/server-scm/src/main/webapps/scm/scm-overview.html
+++ /dev/null
@@ -1,60 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h2>Node counts</h2>
-
-<table class="table table-bordered table-striped" class="col-md-6">
- <tbody>
- <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder">
- <td>{{typestat.key}}</td>
- <td>{{typestat.value}}</td>
- </tr>
- </tbody>
-</table>
-
-<h2>Status</h2>
-<table class="table table-bordered table-striped" class="col-md-6">
- <tbody>
- <tr>
- <td>Client Rpc port</td>
- <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td>
- </tr>
- <tr>
- <td>Datanode Rpc port</td>
- <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td>
- </tr>
- <tr>
- <td>Block Manager: Open containers</td>
- <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td>
- </tr>
- <tr>
- <td>Node Manager: Minimum chill mode nodes</td>
- <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td>
- </tr>
- <tr>
- <td>Node Manager: Out-of-node chill mode</td>
- <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td>
- </tr>
- <tr>
- <td>Node Manager: Chill mode status</td>
- <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td>
- </tr>
- <tr>
- <td>Node Manager: Manual chill mode</td>
- <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td>
- </tr>
- </tbody>
-</table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js
deleted file mode 100644
index bcfa8b7..0000000
--- a/hadoop-hdds/server-scm/src/main/webapps/scm/scm.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
- angular.module('scm', ['ozone', 'nvd3']);
-
- angular.module('scm').component('scmOverview', {
- templateUrl: 'scm-overview.html',
- require: {
- overview: "^overview"
- },
- controller: function ($http) {
- var ctrl = this;
- $http.get("jmx?qry=Hadoop:service=BlockManager,name=*")
- .then(function (result) {
- ctrl.blockmanagermetrics = result.data.beans[0];
- });
- $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo")
- .then(function (result) {
- ctrl.nodemanagermetrics = result.data.beans[0];
- });
-
- var statusSortOrder = {
- "HEALTHY": "a",
- "STALE": "b",
- "DEAD": "c",
- "UNKNOWN": "z",
- "DECOMMISSIONING": "x",
- "DECOMMISSIONED": "y"
- };
- ctrl.nodeOrder = function (v1, v2) {
- //status with non defined sort order will be "undefined"
- return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value])
- }
-
- }
- });
-
-})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-hdds/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml
index 852c81d..37c7d9d 100644
--- a/hadoop-hdds/tools/pom.xml
+++ b/hadoop-hdds/tools/pom.xml
@@ -29,26 +29,18 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop HDDS Tools</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>hdds</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdds-client</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml
index bbd0759..e471710 100644
--- a/hadoop-ozone/client/pom.xml
+++ b/hadoop-ozone/client/pom.xml
@@ -28,16 +28,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Client</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
</dependencies>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml
index 37a2cb9..942576b 100644
--- a/hadoop-ozone/common/pom.xml
+++ b/hadoop-ozone/common/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Common</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 6bf8b01..d173ec6 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -68,6 +68,7 @@ function ozonecmd_case
datanode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-datanode"
;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
@@ -86,42 +87,50 @@ function ozonecmd_case
;;
freon)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.Freon
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
genesis)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
getozoneconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
om)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager
HDFS_OM_OPTS="${HDFS_OM_OPTS} -Dlog4j.configurationFile=${HADOOP_CONF_DIR}/om-audit-log4j2.properties"
HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_OM_OPTS}"
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
;;
sh | shell)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell
- ;;
- noz)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SQLCLI
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-ozone-manager"
;;
scm)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManager'
hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}"
+ OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm"
+
;;
fs)
HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
scmcli)
HADOOP_CLASSNAME=org.apache.hadoop.hdds.scm.cli.SCMCLI
+ OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-tools"
;;
version)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.util.OzoneVersionInfo
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-common"
;;
genconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations
+ OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
*)
HADOOP_CLASSNAME="${subcmd}"
@@ -161,6 +170,7 @@ fi
HADOOP_SUBCMD=$1
shift
+
if hadoop_need_reexec ozone "${HADOOP_SUBCMD}"; then
hadoop_uservar_su ozone "${HADOOP_SUBCMD}" \
"${MYNAME}" \
@@ -180,6 +190,40 @@ else
ozonecmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
+
+#
+# Setting up classpath based on the generate classpath descriptors
+#
+if [ ! "$OZONE_RUN_ARTIFACT_NAME" ]; then
+ echo "ERROR: Ozone components require to set OZONE_RUN_ARTIFACT_NAME to set the classpath"
+ exit -1
+fi
+export HDDS_LIB_JARS_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
+CLASSPATH_FILE="${HADOOP_HDFS_HOME}/share/ozone/classpath/${OZONE_RUN_ARTIFACT_NAME}.classpath"
+if [ ! "$CLASSPATH_FILE" ]; then
+ echo "ERROR: Classpath file descriptor $CLASSPATH_FILE is missing"
+ exit -1
+fi
+# shellcheck disable=SC1090,SC2086
+source $CLASSPATH_FILE
+OIFS=$IFS
+IFS=':'
+# shellcheck disable=SC2154
+for jar in $classpath; do
+ hadoop_add_classpath "$jar"
+done
+hadoop_add_classpath "${HADOOP_HDFS_HOME}/share/ozone/web"
+
+#We need to add the artifact manually as it's not part the generated classpath desciptor
+ARTIFACT_LIB_DIR="${HADOOP_HDFS_HOME}/share/ozone/lib"
+MAIN_ARTIFACT=$(find "$ARTIFACT_LIB_DIR" -name "${OZONE_RUN_ARTIFACT_NAME}-*.jar")
+if [ ! "$MAIN_ARTIFACT" ]; then
+ echo "ERROR: Component jar file $MAIN_ARTIFACT is missing from ${HADOOP_HDFS_HOME}/share/ozone/lib"
+fi
+hadoop_add_classpath "${MAIN_ARTIFACT}"
+IFS=$OIFS
+
+
hadoop_add_client_opts
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh
index 99885ab..3fff7f5 100644
--- a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh
+++ b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh
@@ -19,27 +19,3 @@ if [[ "${HADOOP_SHELL_EXECNAME}" = ozone ]]; then
fi
-## @description Profile for hdds/ozone components.
-## @audience private
-## @stability evolving
-function _ozone_hadoop_classpath
-{
- #
- # get all of the ozone jars+config in the path
- #
-
- if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}"
- fi
-
- if [[ -d "${HADOOP_HDFS_HOME}/${HDDS_DIR}/webapps" ]]; then
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}"
- fi
-
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_LIB_JARS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_LIB_JARS_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}"'/*'
- hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONEFS_DIR}"'/*'
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/datanode/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/datanode/pom.xml b/hadoop-ozone/datanode/pom.xml
new file mode 100644
index 0000000..02995f5
--- /dev/null
+++ b/hadoop-ozone/datanode/pom.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-datanode</artifactId>
+ <name>Apache Hadoop Ozone Datanode</name>
+ <packaging>jar</packaging>
+ <version>0.3.0-SNAPSHOT</version>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-container-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-objectstore-service</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
new file mode 100755
index 0000000..3f102fa
--- /dev/null
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# project.build.directory
+BASEDIR=$1
+
+#hdds.version
+HDDS_VERSION=$2
+
+## @audience private
+## @stability evolving
+function run()
+{
+ declare res
+
+ echo "\$ ${*}"
+ "${@}"
+ res=$?
+ if [[ ${res} != 0 ]]; then
+ echo
+ echo "Failed!"
+ echo
+ exit "${res}"
+ fi
+}
+
+## @audience private
+## @stability evolving
+function findfileindir()
+{
+ declare file="$1"
+ declare dir="${2:-./share}"
+ declare count
+
+ count=$(find "${dir}" -iname "${file}" | wc -l)
+
+ #shellcheck disable=SC2086
+ echo ${count}
+}
+
+
+# shellcheck disable=SC2164
+ROOT=$(cd "${BASEDIR}"/../../..;pwd)
+echo
+echo "Current directory $(pwd)"
+echo
+
+run rm -rf "ozone-${HDDS_VERSION}"
+run mkdir "ozone-${HDDS_VERSION}"
+run cd "ozone-${HDDS_VERSION}"
+run cp -p "${ROOT}/LICENSE.txt" .
+run cp -p "${ROOT}/NOTICE.txt" .
+run cp -p "${ROOT}/README.txt" .
+
+run mkdir -p ./share/hadoop/mapreduce
+run mkdir -p ./share/hadoop/ozone
+run mkdir -p ./share/hadoop/hdds
+run mkdir -p ./share/hadoop/yarn
+run mkdir -p ./share/hadoop/hdfs
+run mkdir -p ./share/hadoop/common
+run mkdir -p ./share/ozone/web
+run mkdir -p ./bin
+run mkdir -p ./sbin
+run mkdir -p ./etc
+run mkdir -p ./libexec
+
+run cp -r "${ROOT}/hadoop-common-project/hadoop-common/src/main/conf" "etc/hadoop"
+
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop" "bin/"
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd" "bin/"
+run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone" "bin/"
+
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh" "libexec/"
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd" "libexec/"
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh" "libexec/"
+run cp "${ROOT}/hadoop-ozone/common/src/main/bin/ozone-config.sh" "libexec/"
+run cp -r "${ROOT}/hadoop-ozone/common/src/main/shellprofile.d" "libexec/"
+
+
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh" "sbin/"
+run cp "${ROOT}/hadoop-common-project/hadoop-common/src/main/bin/workers.sh" "sbin/"
+run cp "${ROOT}/hadoop-ozone/common/src/main/bin/start-ozone.sh" "sbin/"
+run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/"
+
+#shaded ozonefs
+run mkdir -p "./share/hadoop/ozonefs"
+run cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
+
+#shaded datanode service
+run mkdir -p "./share/hadoop/ozoneplugin"
+run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
+
+
+# Optional documentation, could be missing
+cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
+cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/
+
+#Copy docker compose files
+run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" .
+run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching
new file mode 100755
index 0000000..c94e7d0
--- /dev/null
+++ b/hadoop-ozone/dist/dev-support/bin/dist-tar-stitching
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# project.version
+VERSION=$1
+
+# project.build.directory
+BASEDIR=$2
+
+## @audience private
+## @stability evolving
+function run()
+{
+ declare res
+
+ echo "\$ ${*}"
+ "${@}"
+ res=$?
+ if [[ ${res} != 0 ]]; then
+ echo
+ echo "Failed!"
+ echo
+ exit "${res}"
+ fi
+}
+
+run tar -c -f "ozone-${VERSION}.tar" "ozone-${VERSION}"
+run gzip -f "ozone-${VERSION}.tar"
+echo
+echo "Ozone dist tar available at: ${BASEDIR}/ozone-${VERSION}.tar.gz"
+echo
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
new file mode 100644
index 0000000..ba9f14f
--- /dev/null
+++ b/hadoop-ozone/dist/pom.xml
@@ -0,0 +1,184 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone</artifactId>
+ <version>0.3.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-ozone-dist</artifactId>
+ <name>Apache Hadoop Ozone Distribution</name>
+ <packaging>pom</packaging>
+ <version>0.3.0-SNAPSHOT</version>
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-classpath-files</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/ozone-${ozone.version}/share/ozone/classpath
+ </outputDirectory>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-scm</artifactId>
+ <version>${hdds.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-hdds-server-scm.classpath</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-tools</artifactId>
+ <version>${hdds.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-hdds-tools.classpath</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ <version>${ozone.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-ozone-ozone-manager.classpath
+ </destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-tools</artifactId>
+ <version>${ozone.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-ozone-tools.classpath</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ <version>${ozone.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-ozone-common.classpath</destFileName>
+ </artifactItem>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-datanode</artifactId>
+ <version>${ozone.version}</version>
+ <classifier>classpath</classifier>
+ <destFileName>hadoop-ozone-datanode.classpath</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-jars</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/ozone-${ozone.version}/share/ozone/lib
+ </outputDirectory>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${shell-executable}</executable>
+ <workingDirectory>${project.build.directory}</workingDirectory>
+ <arguments>
+ <argument>
+ ${basedir}/dev-support/bin/dist-layout-stitching
+ </argument>
+ <argument>${project.build.directory}</argument>
+ <argument>${hdds.version}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>tar-ozone</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${shell-executable}</executable>
+ <workingDirectory>${project.build.directory}
+ </workingDirectory>
+ <arguments>
+ <argument>${basedir}/dev-support/bin/dist-tar-stitching
+ </argument>
+ <argument>${hdds.version}</argument>
+ <argument>${project.build.directory}</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-scm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-container-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-datanode</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/README.md b/hadoop-ozone/dist/src/main/compose/README.md
new file mode 100644
index 0000000..8189d2c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/README.md
@@ -0,0 +1,51 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Docker cluster definitions
+
+This directory contains multiple docker cluster definitions to start local pseudo cluster with different configuration.
+
+It helps to start local (multi-node like) pseudo cluster with docker and docker-compose and obviously it's not for production.
+
+You may find more information in the specific subdirectories but in generic you can use the following commands:
+
+## Usage
+
+To start a cluster go to a subdirectory and start the cluster:
+
+```
+docker-compose up -d
+```
+
+You can check the logs of all the components with:
+
+```
+docker-compose logs
+```
+
+In case of a problem you can destroy the cluster an delete all the local state with:
+
+```
+docker-compose down
+```
+
+(Note: a simple docker-compose stop may not delete all the local data).
+
+You can scale up and down the components:
+
+```
+docker-compose scale datanode=5
+```
+
+Usually the key webui ports are published on the docker host.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env
new file mode 100644
index 0000000..c437513
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HADOOP_VERSION=3.1.0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
new file mode 100644
index 0000000..b89052d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ namenode:
+ image: flokkr/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9870:9870
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
+ command: ["hdfs", "namenode"]
+ datanode:
+ image: flokkr/hadoop:${HADOOP_VERSION}
+ ports:
+ - 9864
+ volumes:
+ - ../..:/opt/ozone
+ command: ["hdfs","datanode"]
+ environment:
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["ozone","om"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
new file mode 100644
index 0000000..3b2819f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=ozoneManager
+OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=true
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/.env b/hadoop-ozone/dist/src/main/compose/ozone/.env
new file mode 100644
index 0000000..67eed25
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
new file mode 100644
index 0000000..0a6a9d8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-compose.yaml
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
new file mode 100644
index 0000000..f2c8db1
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=ozoneManager
+OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
new file mode 100644
index 0000000..a1e8748
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: apache/hadoop-runner
+ hostname: ozoneManager
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ hadooplast:
+ image: flokkr/hadoop:3.1.0
+ volumes:
+ - ../..:/opt/ozone
+ env_file:
+ - ./docker-config
+ environment:
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
+ command: ["watch","-n","100000","ls"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config
new file mode 100644
index 0000000..3171f08
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-config
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+OZONE-SITE.XML_ozone.om.address=ozoneManager
+OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.scm.heartbeat.interval=3s
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/.env b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env
new file mode 100644
index 0000000..cac418a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/4] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
Posted by bh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
new file mode 100644
index 0000000..527ff41
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/README.md
@@ -0,0 +1,73 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Compose files for local performance tests
+
+This directory contains docker-compose definition for an ozone cluster where
+all the metrics are saved to a prometheus instance.
+
+ Prometheus follows a pull based approach where the metrics are published
+ on a HTTP endpoint.
+
+ Our current approach:
+
+ 1. A Java agent activates a prometheus metrics endpoint in every JVM instance
+ (use `init.sh` to download the agent)
+
+ 2. The Java agent publishes all the jmx parameters in prometheus format AND
+ register the endpoint address to the consul.
+
+ 3. Prometheus polls all the endpoints which are registered to consul.
+
+
+
+## How to use
+
+First of all download the required Java agent with running `./init.sh`
+
+After that you can start the cluster with docker-compose:
+
+```
+docker-compose up -d
+```
+
+After a while the cluster will be started. You can check the ozone web ui-s:
+
+https://localhost:9874
+https://localhost:9876
+
+You can also scale up the datanodes:
+
+```
+docker-compose scale datanode=3
+```
+
+Freon (Ozone test generator tool) is not part of docker-compose by default,
+you can activate it using `compose-all.sh` instead of `docker-compose`:
+
+```
+compose-all.sh up -d
+```
+
+Now Freon is running. Let's try to check the metrics from the local Prometheus:
+
+http://localhost:9090/graph
+
+Example queries:
+
+```
+Hadoop_OzoneManager_NumKeyCommits
+rate(Hadoop_OzoneManager_NumKeyCommits[10m])
+rate(Hadoop_Ozone_BYTES_WRITTEN[10m])
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh
new file mode 100755
index 0000000..82ab8b3
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/compose-all.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
new file mode 100644
index 0000000..60bdc4a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose-freon.yaml
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ freon:
+ image: apache/hadoop-runner
+ volumes:
+ - ../../ozone:/opt/hadoop
+ - ./jmxpromo.jar:/opt/jmxpromo.jar
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","freon"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
new file mode 100644
index 0000000..12b28bb
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-compose.yaml
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ - ./jmxpromo.jar:/opt/jmxpromo.jar
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ - ./jmxpromo.jar:/opt/jmxpromo.jar
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: apache/hadoop-runner
+ volumes:
+ - ../..:/opt/hadoop
+ - ./jmxpromo.jar:/opt/jmxpromo.jar
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
+ consul:
+ image: consul
+ command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"]
+ ports:
+ - 8500:8500
+ prometheus:
+ image: prom/prometheus
+ volumes:
+ - "./prometheus.yml:/etc/prometheus.yml"
+ command: ["--config.file","/etc/prometheus.yml"]
+ ports:
+ - 9090:9090
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
new file mode 100644
index 0000000..1ed1169
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=ozoneManager
+OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh b/hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh
new file mode 100755
index 0000000..cf25398
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/init.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+EXPORTER_FILE="$DIR/jmxpromo.jar"
+if [ ! -f "$EXPORTER_FILE" ]; then
+ wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml
new file mode 100644
index 0000000..80aa520
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozoneperf/prometheus.yml
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+global:
+ scrape_interval: 15s # By default, scrape targets every 15 seconds.
+
+scrape_configs:
+ - job_name: jmxexporter
+ consul_sd_configs:
+ - server: consul:8500
+ services:
+ - jmxexporter
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys
new file mode 100644
index 0000000..ae39052
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/authorized_keys
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config
new file mode 100644
index 0000000..6506916
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/config
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Host *
+ UserKnownHostsFile /dev/null
+ StrictHostKeyChecking no
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment
new file mode 100644
index 0000000..5685453
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/environment
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa
new file mode 100644
index 0000000..6632ce5
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4BJi6WJuAa1ratShvYYWVwmYBqxE57btHjU6NtVN1SnPZx/f
+6LezOpQGsLBXE/bl7uG+fD05Z378B/0wE5QhYwvJ9Ge0jsfhVOi90p/FEYfR2l+C
+9LRfLvO6AgA+HQa1BoYQd+norh/XQQVpcukSn32Cb642rW3OxsQTv3uTklDLapPw
+0lSpLJgYWcbrayorwjYw4rIfVYhiH/G+ckvzJRBpmlSgKYpNXyeYjpZaDPxoKQet
+GFf4W6nwnnG1oHojNYCvdoHTnAP1sdQu34/AUiOYg0hOnmXhfNIDN+NtZb8jw1sl
+YRWhm5GuK1wV/32/UEkQQPUU6oUSgBJgCH+PRwIDAQABAoIBAQDI1TH6ZNKchkck
+9XgSWsBjOqIcOQN5fCeDT8nho8WjLVpL3/Hcr+ngsxRcAXHK3xyvw33r9SQic1qJ
+/pC8u6RBFivo95qJ7vU0GXcp9TG4yLd6tui1U4WMm784U+dYNM7EDh1snSaECt3v
+1V3yNJ0QfnoOh2NShn0zAkOA+M4H8Nx2SudMCsjcbK9+fYxzW3hX+sJpMKdjG1HW
+DUz+I7cW7t0EGaVrgVSV+eR58LiXu+14YDNMrySiejB4nD2sKrx93XgiCBECCsBN
+GLQGJCztaXoAY+5Kf/aJ9EEf2wBF3GecRk+XIAd87PeDmeahLQAVkAJ/rD1vsKFs
+8kWi6CrhAoGBAP7leG/dMBhlfvROpBddIfWm2i8B+oZiAlSjdYGz+/ZhUaByXk18
+pupMGiiMQR1ziPnEg0gNgR2ZkH54qrXPn5WcQa4rlSEtUsZEp5v5WblhfX2QwKzY
+G/uhA+mB7wXpQkSmXo0LclfPF2teROQrG1OyfWkWbxFH4i3+em7sL95jAoGBAOEK
+v+wscqkMLW7Q8ONbWMCCBlmMHr6baB3VDCYZx25lr+GIF5zmJJFTmF2rq2VSAlts
+qx1AGmaUSo78kC5FuJvSNTL6a1Us5ucdthQZM3N8pAz+OAE+QEU+BsdA27yAh3tO
+yKDsMFNHKtXcgy5LeB5gzENLlNyw2jgkRv2Ef77NAoGAVH8DHqoHEH9Mx3XuRWR1
+JnaqKx0PzE5fEWmiQV3Fr5XxNivTgQJKXq7dmQVtbHLpPErdbhwz6fkHAjXD+UMb
+VsAWscL2y6m3n8wQd87/5EkiDWbXyDRXimGE53pQHviFJDa2bzEVNXCMBeaZFb4I
+cAViN1zdcrAOlUqfkXewIpsCgYB8wsXl/DpRB+RENGfn0+OfTjaQ/IKq72NIbq1+
+jfondQ6N/TICFQEe5HZrL9okoNOXteYjoD9CsWGoZdLVJGgVUvOVYImSvgMBDFK+
+T75bfzU/0sxfvBBLkviVDJsFpUf3D5VgybB86s6Po+HCD6r3RHjZshRESXOhflMx
+B3z+3QKBgE2Lwo0DuwUGkm6k8psyn3x8EiXNsiNw12cojicFTyKUYLHxMBeVbCLW
+3j3pxSggJgRuBLLzixUHbHp91r2ARTy28naK7R/la8yKVqK6ojcikN2mQsCHYtwB
+nuFwXr42ytn6G+9Wn4xT64tGjRCqyZn0/v0XsPjVCyrZ6G7EtNHP
+-----END RSA PRIVATE KEY-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
new file mode 100644
index 0000000..ae39052
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/.ssh/id_rsa.pub
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDgEmLpYm4BrWtq1KG9hhZXCZgGrETntu0eNTo21U3VKc9nH9/ot7M6lAawsFcT9uXu4b58PTlnfvwH/TATlCFjC8n0Z7SOx+FU6L3Sn8URh9HaX4L0tF8u87oCAD4dBrUGhhB36eiuH9dBBWly6RKffYJvrjatbc7GxBO/e5OSUMtqk/DSVKksmBhZxutrKivCNjDish9ViGIf8b5yS/MlEGmaVKApik1fJ5iOlloM/GgpB60YV/hbqfCecbWgeiM1gK92gdOcA/Wx1C7fj8BSI5iDSE6eZeF80gM3421lvyPDWyVhFaGbka4rXBX/fb9QSRBA9RTqhRKAEmAIf49H hadoop@cdae967fa87a
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile
new file mode 100644
index 0000000..3bdcb0c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/Dockerfile
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM apache/hadoop-runner
+RUN sudo apt-get update && sudo apt-get install -y openssh-server
+
+RUN sudo mkdir -p /run/sshd
+RUN sudo sed -i "s/.*UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
+RUN sudo sed -i "s/.*PermitUserEnvironment.*/PermitUserEnvironment yes/g" /etc/ssh/sshd_config
+RUN sudo sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+#/opt/hadoop is mounted, we can't use it as a home
+RUN sudo usermod -d /opt hadoop
+ADD .ssh /opt/.ssh
+RUN sudo chown -R hadoop /opt/.ssh
+RUN sudo chown hadoop /opt
+RUN sudo chmod 600 /opt/.ssh/*
+RUN sudo chmod 700 /opt/.ssh
+
+RUN sudo sh -c 'echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/" >> /etc/profile'
+CMD ["sudo","/usr/sbin/sshd","-D"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md b/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
new file mode 100644
index 0000000..2531fa4
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/README.md
@@ -0,0 +1,38 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# start-ozone environment
+
+This is an example environment to use/test `./sbin/start-ozone.sh` and `./sbin/stop-ozone.sh` scripts.
+
+There are ssh connections between the containers and the start/stop scripts could handle the start/stop process
+similar to a real cluster.
+
+To use it, first start the cluster:
+
+```
+docker-copmose up -d
+```
+
+After a successfull startup (which starts only the ssh daemons) you can start ozone:
+
+```
+./start.sh
+```
+
+Check it the java processes are started:
+
+```
+./ps.sh
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
new file mode 100644
index 0000000..62f1163
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-compose.yaml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ datanode:
+ build: .
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9864
+ env_file:
+ - ./docker-config
+ om:
+ build: .
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9874:9874
+ env_file:
+ - ./docker-config
+ scm:
+ build: .
+ volumes:
+ - ../..:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
new file mode 100644
index 0000000..1afec73
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64/
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=true
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
new file mode 100755
index 0000000..d5e2c38
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/ps.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker-compose ps -q | xargs -n1 -I CONTAINER docker exec CONTAINER ps xa
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
new file mode 100755
index 0000000..3358b07
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/start.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker-compose ps | grep datanode | awk '{print $1}' | xargs -n1 docker inspect --format '{{ .Config.Hostname }}' > ../../etc/hadoop/workers
+docker-compose exec scm /opt/hadoop/bin/ozone scm -init
+docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
+#We need a running SCM for om objectstore creation
+#TODO create a utility to wait for the startup
+sleep 10
+docker-compose exec om /opt/hadoop/bin/ozone om -createObjectStore
+docker-compose exec scm /opt/hadoop/sbin/start-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh b/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
new file mode 100755
index 0000000..a3ce08a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonescripts/stop.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker-compose exec scm /opt/hadoop/sbin/stop-ozone.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/ozone/README.txt
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/ozone/README.txt b/hadoop-ozone/dist/src/main/ozone/README.txt
new file mode 100644
index 0000000..6bbd83f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/ozone/README.txt
@@ -0,0 +1,51 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+This is the distribution of Apache Hadoop Ozone.
+
+Ozone is a submodule of Hadoop with separated release cycle. For more information, check
+
+ http://ozone.hadoop.apache.org
+
+ and
+
+ https://cwiki.apache.org/confluence/display/HADOOP/Ozone+Contributor+Guide
+
+For more information about Hadoop, check:
+
+ http://hadoop.apache.org
+
+This distribution includes cryptographic software. The country in
+which you currently reside may have restrictions on the import,
+possession, use, and/or re-export to another country, of
+encryption software. BEFORE using any encryption software, please
+check your country's laws, regulations and policies concerning the
+import, possession, or use, and re-export of encryption software, to
+see if this is permitted. See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and
+Security (BIS), has classified this software as Export Commodity
+Control Number (ECCN) 5D002.C.1, which includes information security
+software using or performing cryptographic functions with asymmetric
+algorithms. The form and manner of this Apache Software Foundation
+distribution makes it eligible for export under the License Exception
+ENC Technology Software Unrestricted (TSU) exception (see the BIS
+Export Administration Regulations, Section 740.13) for both object
+code and source code.
+
+The following provides more details on the included cryptographic
+software:
+ Hadoop Core uses the SSL libraries from the Jetty project written
+by mortbay.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/README.md b/hadoop-ozone/dist/src/main/smoketest/README.md
new file mode 100644
index 0000000..c521a54
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/README.md
@@ -0,0 +1,30 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+## Ozone Acceptance Tests
+
+This directory contains a [robotframework](http://robotframework.org/) based test suite for Ozone to make it easier to check the current state of the package.
+
+You can run in in any environment after [installing](https://github.com/robotframework/robotframework/blob/master/INSTALL.rst)
+
+```
+cd $DIRECTORY_OF_OZONE
+robot smoketest/basic
+```
+
+The argument of the `robot` could be any robot file or directory.
+
+The current configuration in the robot files (hostnames, ports) are adjusted for the docker-based setup but you can easily modify it for any environment.
+
+The `./test.sh` in this directory can start multiple type of clusters (ozone standalone or ozone + hdfs) and execute the test framework with all of the clusters.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
new file mode 100644
index 0000000..a69450d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/basic.robot
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoketest ozone cluster startup
+Library OperatingSystem
+Resource ../commonlib.robot
+
+*** Variables ***
+${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
+${DATANODE_HOST} localhost
+
+
+*** Test Cases ***
+
+Test rest interface
+ ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
+ Should contain ${result} 201 Created
+ ${result} = Execute curl -i -X POST ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
+ Should contain ${result} 201 Created
+ ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1/bucket1"
+ Should contain ${result} 200 OK
+ ${result} = Execute curl -i -X DELETE ${COMMON_RESTHEADER} "http://${DATANODE_HOST}:9880/volume1"
+ Should contain ${result} 200 OK
+
+Check webui static resources
+ ${result} = Execute curl -s -I http://scm:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
+ Should contain ${result} 200
+ ${result} = Execute curl -s -I http://ozoneManager:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
+ Should contain ${result} 200
+
+Start freon testing
+ ${result} = Execute ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
+ Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
+ Should Not Contain ${result} ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
new file mode 100644
index 0000000..14a5761
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/basic/ozone-shell.robot
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Test ozone shell CLI usage
+Library OperatingSystem
+Resource ../commonlib.robot
+Test Timeout 2 minute
+
+*** Variables ***
+
+*** Test Cases ***
+RestClient without http port
+ Test ozone shell http:// ozoneManager restwoport
+
+RestClient with http port
+ Test ozone shell http:// ozoneManager:9874 restwport
+
+RestClient without host name
+ Test ozone shell http:// ${EMPTY} restwohost
+
+RpcClient with port
+ Test ozone shell o3:// ozoneManager:9862 rpcwoport
+
+RpcClient without host
+ Test ozone shell o3:// ${EMPTY} rpcwport
+
+RpcClient without scheme
+ Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme
+
+
+*** Keywords ***
+Test ozone shell
+ [arguments] ${protocol} ${server} ${volume}
+ ${result} = Execute ozone sh volume create ${protocol}${server}/${volume} --user bilbo --quota 100TB --root
+ Should not contain ${result} Failed
+ Should contain ${result} Creating Volume: ${volume}
+ ${result} = Execute ozone sh volume list ${protocol}${server}/ --user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
+ Should contain ${result} createdOn
+ ${result} = Execute ozone sh volume list --user bilbo | grep -Ev 'Removed|DEBUG|ERROR|INFO|TRACE|WARN' | jq -r '.[] | select(.volumeName=="${volume}")'
+ Should contain ${result} createdOn
+ Execute ozone sh volume update ${protocol}${server}/${volume} --user bill --quota 10TB
+ ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
+ Should Be Equal ${result} bill
+ ${result} = Execute ozone sh volume info ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .quota | .size'
+ Should Be Equal ${result} 10
+ Execute ozone sh bucket create ${protocol}${server}/${volume}/bb1
+ ${result} = Execute ozone sh bucket info ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .storageType'
+ Should Be Equal ${result} DISK
+ ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --addAcl user:frodo:rw,group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="samwise") | .type'
+ Should Be Equal ${result} GROUP
+ ${result} = Execute ozone sh bucket update ${protocol}${server}/${volume}/bb1 --removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
+ Should Be Equal ${result} USER
+ ${result} = Execute ozone sh bucket list ${protocol}${server}/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
+ Should Be Equal ${result} ${volume}
+ Run Keyword Test key handling ${protocol} ${server} ${volume}
+ Execute ozone sh bucket delete ${protocol}${server}/${volume}/bb1
+ Execute ozone sh volume delete ${protocol}${server}/${volume} --user bilbo
+
+Test key handling
+ [arguments] ${protocol} ${server} ${volume}
+ Execute ozone sh key put ${protocol}${server}/${volume}/bb1/key1 /opt/hadoop/NOTICE.txt
+ Execute rm -f NOTICE.txt.1
+ Execute ozone sh key get ${protocol}${server}/${volume}/bb1/key1 NOTICE.txt.1
+ Execute ls -l NOTICE.txt.1
+ ${result} = Execute ozone sh key info ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
+ Should contain ${result} createdOn
+ ${result} = Execute ozone sh key list ${protocol}${server}/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
+ Should Be Equal ${result} key1
+ Execute ozone sh key delete ${protocol}${server}/${volume}/bb1/key1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/commonlib.robot b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
new file mode 100644
index 0000000..e2620fa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/commonlib.robot
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Keywords ***
+
+
+Execute
+ [arguments] ${command}
+ ${rc} ${output} = Run And Return Rc And Output ${command}
+ Log ${output}
+ Should Be Equal As Integers ${rc} 0
+ [return] ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
new file mode 100644
index 0000000..fb7b98c
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Ozonefs test
+Library OperatingSystem
+Resource ../commonlib.robot
+
+*** Variables ***
+
+
+*** Test Cases ***
+Create volume and bucket
+ Execute ozone sh volume create http://ozoneManager/fstest --user bilbo --quota 100TB --root
+ Execute ozone sh bucket create http://ozoneManager/fstest/bucket1
+
+Check volume from ozonefs
+ ${result} = Execute ozone fs -ls o3://bucket1.fstest/
+
+Create directory from ozonefs
+ Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
+ ${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
new file mode 100755
index 0000000..534bbb7
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+execute_tests(){
+ COMPOSE_FILE=$DIR/../compose/$1/docker-compose.yaml
+ TESTS=$2
+ echo "Executing test ${TESTS[*]} with $COMPOSE_FILE"
+ docker-compose -f "$COMPOSE_FILE" down
+ docker-compose -f "$COMPOSE_FILE" up -d
+ docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get update
+ docker-compose -f "$COMPOSE_FILE" exec datanode sudo apt-get install -y python-pip
+ docker-compose -f "$COMPOSE_FILE" exec datanode sudo pip install robotframework
+ for TEST in "${TESTS[@]}"; do
+ set +e
+ docker-compose -f "$COMPOSE_FILE" exec datanode python -m robot "smoketest/$TEST"
+ set -e
+ done
+ if [ "$KEEP_RUNNING" = false ]; then
+ docker-compose -f "$COMPOSE_FILE" down
+ fi
+}
+RUN_ALL=true
+KEEP_RUNNING=false
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+ --env)
+ DOCKERENV="$2"
+ RUN_ALL=false
+ shift # past argument
+ shift # past value
+ ;;
+ --keep)
+ KEEP_RUNNING=true
+ shift # past argument
+ ;;
+ --help|-h|-help)
+ cat << EOF
+
+ Acceptance test executor for ozone.
+
+ This is a lightweight test executor for ozone.
+
+ You can run it with
+
+ ./test.sh
+
+ Which executes all the tests in all the available environments.
+
+ Or you can run manually one test with
+
+ ./test.sh --keep --env ozone-hdfs basic
+
+ --keep means that docker cluster won't be stopped after the test (optional)
+ --env defines the subdirectory under the compose dir
+ The remaining parameters define the test suites under smoketest dir.
+ Could be any directory or robot file relative to the smoketest dir.
+EOF
+ exit 0
+ ;;
+ *)
+ POSITIONAL+=("$1") # save it in an array for later
+ shift # past argument
+ ;;
+esac
+done
+
+if [ "$RUN_ALL" = true ]; then
+#
+# This is the definition of the ozone acceptance test suite
+#
+# We select the test suites and execute them on multiple type of clusters
+#
+ DEFAULT_TESTS=("basic")
+ execute_tests ozone "${DEFAULT_TESTS[@]}"
+ TESTS=("ozonefs")
+ execute_tests ozonefs "${TESTS[@]}"
+
+else
+ execute_tests "$DOCKERENV" "${POSITIONAL[@]}"
+fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/docs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/pom.xml b/hadoop-ozone/docs/pom.xml
index 64d0ec8..d8edd15 100644
--- a/hadoop-ozone/docs/pom.xml
+++ b/hadoop-ozone/docs/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Documentation</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index 993e91f..bed2fce 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -28,37 +28,23 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Integration Tests</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-ozone-manager</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-objectstore-service</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-ozone-tools</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
deleted file mode 100644
index a2df50d..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.freon;
-
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Tests Freon, with MiniOzoneCluster and validate data.
- */
-public class TestDataValidate {
-
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration conf;
-
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- */
- @BeforeClass
- public static void init() throws Exception {
- conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test
- public void ratisTestLargeKey() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setKeySize(20971520);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
- Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
- }
-
- @Test
- public void standaloneTestLargeKey() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(1);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(1);
- randomKeyGenerator.setKeySize(20971520);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
- Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
- }
-
- @Test
- public void validateWriteTest() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setValidateWrites(true);
- randomKeyGenerator.call();
- Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
- Assert.assertTrue(randomKeyGenerator.getValidateWrites());
- Assert.assertNotEquals(0, randomKeyGenerator.getTotalKeysValidated());
- Assert.assertNotEquals(0, randomKeyGenerator
- .getSuccessfulValidationCount());
- Assert.assertEquals(0, randomKeyGenerator
- .getUnsuccessfulValidationCount());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
deleted file mode 100644
index d21d399..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.freon;
-
-import org.apache.hadoop.hdds.client.ReplicationFactor;
-import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Tests Freon, with MiniOzoneCluster.
- */
-public class TestRandomKeyGenerator {
-
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration conf;
-
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- */
- @BeforeClass
- public static void init() throws Exception {
- conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterClass
- public static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test
- public void defaultTest() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(2);
- randomKeyGenerator.setNumOfBuckets(5);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.call();
- Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
- Assert.assertEquals(10240 - 36, randomKeyGenerator.getKeyValueLength());
- }
-
- @Test
- public void multiThread() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(10);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.setKeySize(10240);
- randomKeyGenerator.call();
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
- }
-
- @Test
- public void ratisTest3() throws Exception {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
- randomKeyGenerator.setNumOfVolumes(10);
- randomKeyGenerator.setNumOfBuckets(1);
- randomKeyGenerator.setNumOfKeys(10);
- randomKeyGenerator.setNumOfThreads(10);
- randomKeyGenerator.setKeySize(10240);
- randomKeyGenerator.setFactor(ReplicationFactor.THREE);
- randomKeyGenerator.setType(ReplicationType.RATIS);
- randomKeyGenerator.call();
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
- Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
deleted file mode 100644
index a3ff6c8..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.om;
-
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.scm.cli.SQLCLI;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * This class tests the CLI that transforms om.db into SQLite DB files.
- */
-@RunWith(Parameterized.class)
-public class TestOmSQLCli {
- private MiniOzoneCluster cluster = null;
- private StorageHandler storageHandler;
- private UserArgs userArgs;
- private OzoneConfiguration conf;
- private SQLCLI cli;
-
- private String userName = "userTest";
- private String adminName = "adminTest";
- private String volumeName0 = "volumeTest0";
- private String volumeName1 = "volumeTest1";
- private String bucketName0 = "bucketTest0";
- private String bucketName1 = "bucketTest1";
- private String bucketName2 = "bucketTest2";
- private String keyName0 = "key0";
- private String keyName1 = "key1";
- private String keyName2 = "key2";
- private String keyName3 = "key3";
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- // Uncomment the below line if we support leveldb in future.
- //{OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
- });
- }
-
- private String metaStoreType;
-
- public TestOmSQLCli(String type) {
- metaStoreType = type;
- }
-
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- * @throws IOException
- */
- @Before
- public void setup() throws Exception {
- conf = new OzoneConfiguration();
- cluster = MiniOzoneCluster.newBuilder(conf).build();
- cluster.waitForClusterToBeReady();
- storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
- userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
- null, null, null, null);
- cluster.waitForClusterToBeReady();
-
- VolumeArgs createVolumeArgs0 = new VolumeArgs(volumeName0, userArgs);
- createVolumeArgs0.setUserName(userName);
- createVolumeArgs0.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs0);
- VolumeArgs createVolumeArgs1 = new VolumeArgs(volumeName1, userArgs);
- createVolumeArgs1.setUserName(userName);
- createVolumeArgs1.setAdminName(adminName);
- storageHandler.createVolume(createVolumeArgs1);
-
- BucketArgs bucketArgs0 = new BucketArgs(volumeName0, bucketName0, userArgs);
- storageHandler.createBucket(bucketArgs0);
- BucketArgs bucketArgs1 = new BucketArgs(volumeName1, bucketName1, userArgs);
- storageHandler.createBucket(bucketArgs1);
- BucketArgs bucketArgs2 = new BucketArgs(volumeName0, bucketName2, userArgs);
- storageHandler.createBucket(bucketArgs2);
-
- KeyArgs keyArgs0 =
- new KeyArgs(volumeName0, bucketName0, keyName0, userArgs);
- keyArgs0.setSize(100);
- KeyArgs keyArgs1 =
- new KeyArgs(volumeName1, bucketName1, keyName1, userArgs);
- keyArgs1.setSize(200);
- KeyArgs keyArgs2 =
- new KeyArgs(volumeName0, bucketName2, keyName2, userArgs);
- keyArgs2.setSize(300);
- KeyArgs keyArgs3 =
- new KeyArgs(volumeName0, bucketName2, keyName3, userArgs);
- keyArgs3.setSize(400);
-
- OutputStream stream = storageHandler.newKeyWriter(keyArgs0);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs1);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs2);
- stream.close();
- stream = storageHandler.newKeyWriter(keyArgs3);
- stream.close();
-
- cluster.getOzoneManager().stop();
- cluster.getStorageContainerManager().stop();
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
- cli = new SQLCLI(conf);
- }
-
- @After
- public void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- // After HDDS-357, we have to fix SQLCli.
- // TODO: fix SQLCli
- @Ignore
- @Test
- public void testOmDB() throws Exception {
- String dbOutPath = GenericTestUtils.getTempPath(
- UUID.randomUUID() + "/out_sql.db");
-
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
- String dbPath = dbRootPath + "/" + OM_DB_NAME;
- String[] args = {"-p", dbPath, "-o", dbOutPath};
-
- cli.run(args);
-
- Connection conn = connectDB(dbOutPath);
- String sql = "SELECT * FROM volumeList";
- ResultSet rs = executeQuery(conn, sql);
- List<String> expectedValues =
- new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
- while (rs.next()) {
- String userNameRs = rs.getString("userName");
- String volumeNameRs = rs.getString("volumeName");
- assertEquals(userName, userNameRs.substring(1));
- assertTrue(expectedValues.remove(volumeNameRs));
- }
- assertEquals(0, expectedValues.size());
-
- sql = "SELECT * FROM volumeInfo";
- rs = executeQuery(conn, sql);
- expectedValues =
- new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
- while (rs.next()) {
- String adName = rs.getString("adminName");
- String ownerName = rs.getString("ownerName");
- String volumeName = rs.getString("volumeName");
- assertEquals(adminName, adName);
- assertEquals(userName, ownerName);
- assertTrue(expectedValues.remove(volumeName));
- }
- assertEquals(0, expectedValues.size());
-
- sql = "SELECT * FROM aclInfo";
- rs = executeQuery(conn, sql);
- expectedValues =
- new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
- while (rs.next()) {
- String adName = rs.getString("adminName");
- String ownerName = rs.getString("ownerName");
- String volumeName = rs.getString("volumeName");
- String type = rs.getString("type");
- String uName = rs.getString("userName");
- String rights = rs.getString("rights");
- assertEquals(adminName, adName);
- assertEquals(userName, ownerName);
- assertEquals("USER", type);
- assertEquals(userName, uName);
- assertEquals("READ_WRITE", rights);
- assertTrue(expectedValues.remove(volumeName));
- }
- assertEquals(0, expectedValues.size());
-
- sql = "SELECT * FROM bucketInfo";
- rs = executeQuery(conn, sql);
- HashMap<String, String> expectedMap = new HashMap<>();
- expectedMap.put(bucketName0, volumeName0);
- expectedMap.put(bucketName2, volumeName0);
- expectedMap.put(bucketName1, volumeName1);
- while (rs.next()) {
- String volumeName = rs.getString("volumeName");
- String bucketName = rs.getString("bucketName");
- boolean versionEnabled = rs.getBoolean("versionEnabled");
- String storegeType = rs.getString("storageType");
- assertEquals(volumeName, expectedMap.remove(bucketName));
- assertFalse(versionEnabled);
- assertEquals("DISK", storegeType);
- }
- assertEquals(0, expectedMap.size());
-
- sql = "SELECT * FROM keyInfo";
- rs = executeQuery(conn, sql);
- HashMap<String, List<String>> expectedMap2 = new HashMap<>();
- // no data written, data size will be 0
- expectedMap2.put(keyName0,
- Arrays.asList(volumeName0, bucketName0, "0"));
- expectedMap2.put(keyName1,
- Arrays.asList(volumeName1, bucketName1, "0"));
- expectedMap2.put(keyName2,
- Arrays.asList(volumeName0, bucketName2, "0"));
- expectedMap2.put(keyName3,
- Arrays.asList(volumeName0, bucketName2, "0"));
- while (rs.next()) {
- String volumeName = rs.getString("volumeName");
- String bucketName = rs.getString("bucketName");
- String keyName = rs.getString("keyName");
- int dataSize = rs.getInt("dataSize");
- List<String> vals = expectedMap2.remove(keyName);
- assertNotNull(vals);
- assertEquals(vals.get(0), volumeName);
- assertEquals(vals.get(1), bucketName);
- assertEquals(vals.get(2), Integer.toString(dataSize));
- }
- assertEquals(0, expectedMap2.size());
-
- conn.close();
- Files.delete(Paths.get(dbOutPath));
- }
-
- private ResultSet executeQuery(Connection conn, String sql)
- throws SQLException {
- Statement stmt = conn.createStatement();
- return stmt.executeQuery(sql);
- }
-
- private Connection connectDB(String dbPath) throws Exception {
- Class.forName("org.sqlite.JDBC");
- String connectPath =
- String.format("jdbc:sqlite:%s", dbPath);
- return DriverManager.getConnection(connectPath);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
deleted file mode 100644
index 4026348..0000000
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright containerOwnership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
-import org.apache.hadoop.hdds.scm.container.ContainerMapping;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.scm.cli.SQLCLI;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.KB;
-import static org.junit.Assert.assertEquals;
-
-/**
- * This class tests the CLI that transforms container into SQLite DB files.
- */
-@RunWith(Parameterized.class)
-public class TestContainerSQLCli {
-
- private EventQueue eventQueue;
-
- @Parameterized.Parameters
- public static Collection<Object[]> data() {
- return Arrays.asList(new Object[][] {
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
- {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
- });
- }
-
- private static String metaStoreType;
-
- public TestContainerSQLCli(String type) {
- metaStoreType = type;
- }
-
- private static SQLCLI cli;
-
- private MiniOzoneCluster cluster;
- private OzoneConfiguration conf;
- private String datanodeIpAddress;
-
- private ContainerMapping mapping;
- private NodeManager nodeManager;
- private BlockManagerImpl blockManager;
-
- private HashMap<Long, Long> blockContainerMap;
-
- private final static long DEFAULT_BLOCK_SIZE = 4 * KB;
- private static HddsProtos.ReplicationFactor factor;
- private static HddsProtos.ReplicationType type;
- private static final String CONTAINER_OWNER = "OZONE";
-
-
- @Before
- public void setup() throws Exception {
- blockContainerMap = new HashMap<>();
-
- conf = new OzoneConfiguration();
- conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 2);
- conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
- if(conf.getBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT)){
- factor = HddsProtos.ReplicationFactor.THREE;
- type = HddsProtos.ReplicationType.RATIS;
- } else {
- factor = HddsProtos.ReplicationFactor.ONE;
- type = HddsProtos.ReplicationType.STAND_ALONE;
- }
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2).build();
- cluster.waitForClusterToBeReady();
- datanodeIpAddress = cluster.getHddsDatanodes().get(0)
- .getDatanodeDetails().getIpAddress();
- cluster.getOzoneManager().stop();
- cluster.getStorageContainerManager().stop();
- eventQueue = new EventQueue();
- nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
- mapping = new ContainerMapping(conf, nodeManager, 128,
- eventQueue);
- blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue);
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
- eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
- GenericTestUtils.waitFor(() -> {
- return !blockManager.isScmInChillMode();
- }, 10, 1000 * 15);
- // blockManager.allocateBlock() will create containers if there is none
- // stored in levelDB. The number of containers to create is the value of
- // OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2.
- // so the first allocateBlock() will create two containers. A random one
- // is assigned for the block.
-
- // loop until both the two datanodes are up, try up to about 4 seconds.
- for (int c = 0; c < 40; c++) {
- if (nodeManager.getAllNodes().size() == 2) {
- break;
- }
- Thread.sleep(100);
- }
- assertEquals(2, nodeManager.getAllNodes().size());
- AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type,
- factor, CONTAINER_OWNER);
- blockContainerMap.put(ab1.getBlockID().getLocalID(),
- ab1.getBlockID().getContainerID());
-
- AllocatedBlock ab2;
- // we want the two blocks on the two provisioned containers respectively,
- // however blockManager picks containers randomly, keep retry until we
- // assign the second block to the other container. This seems to be the only
- // way to get the two containers.
- // although each retry will create a block and assign to a container. So
- // the size of blockContainerMap will vary each time the test is run.
- while (true) {
- ab2 = blockManager
- .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER);
- blockContainerMap.put(ab2.getBlockID().getLocalID(),
- ab2.getBlockID().getContainerID());
- if (ab1.getBlockID().getContainerID() !=
- ab2.getBlockID().getContainerID()) {
- break;
- }
- }
-
- blockManager.close();
- mapping.close();
- nodeManager.close();
-
- conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
- cli = new SQLCLI(conf);
-
- }
-
- @After
- public void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @Test
- public void testConvertContainerDB() throws Exception {
- String dbOutPath = GenericTestUtils.getTempPath(
- UUID.randomUUID() + "/out_sql.db");
- // TODO : the following will fail due to empty Datanode list, need to fix.
- //String dnUUID = cluster.getDataNodes().get(0).getUuid();
- String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
- String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB;
- String[] args = {"-p", dbPath, "-o", dbOutPath};
- Connection conn;
- String sql;
- ResultSet rs;
-
- cli.run(args);
-
- //verify the sqlite db
- // only checks the container names are as expected. Because other fields
- // such as datanode UUID are generated randomly each time
- conn = connectDB(dbOutPath);
- sql = "SELECT * FROM containerInfo";
- rs = executeQuery(conn, sql);
- ArrayList<Long> containerIDs = new ArrayList<>();
- while (rs.next()) {
- containerIDs.add(rs.getLong("containerID"));
- //assertEquals(dnUUID, rs.getString("leaderUUID"));
- }
- /* TODO: fix this later when the SQLCLI is fixed.
- assertTrue(containerIDs.size() == 2 &&
- containerIDs.contains(pipeline1.getContainerName()) &&
- containerIDs.contains(pipeline2.getContainerName()));
-
- sql = "SELECT * FROM containerMembers";
- rs = executeQuery(conn, sql);
- containerIDs = new ArrayList<>();
- while (rs.next()) {
- containerIDs.add(rs.getLong("containerID"));
- //assertEquals(dnUUID, rs.getString("datanodeUUID"));
- }
- assertTrue(containerIDs.size() == 2 &&
- containerIDs.contains(pipeline1.getContainerName()) &&
- containerIDs.contains(pipeline2.getContainerName()));
-
- sql = "SELECT * FROM datanodeInfo";
- rs = executeQuery(conn, sql);
- int count = 0;
- while (rs.next()) {
- assertEquals(datanodeIpAddress, rs.getString("ipAddress"));
- //assertEquals(dnUUID, rs.getString("datanodeUUID"));
- count += 1;
- }
- // the two containers maybe on the same datanode, maybe not.
- int expected = pipeline1.getLeader().getUuid().equals(
- pipeline2.getLeader().getUuid())? 1 : 2;
- assertEquals(expected, count);
- */
- Files.delete(Paths.get(dbOutPath));
- }
-
- private ResultSet executeQuery(Connection conn, String sql)
- throws SQLException {
- Statement stmt = conn.createStatement();
- return stmt.executeQuery(sql);
- }
-
- private Connection connectDB(String dbPath) throws Exception {
- Class.forName("org.sqlite.JDBC");
- String connectPath =
- String.format("jdbc:sqlite:%s", dbPath);
- return DriverManager.getConnection(connectPath);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org