You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/09/24 18:46:08 UTC
[44/50] [abbrv] hadoop git commit: HDDS-447. Separate ozone-dist and
hadoop-dist projects with real classpath separation. Contributed by Elek
Marton.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/objectstore-service/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/pom.xml b/hadoop-ozone/objectstore-service/pom.xml
index b20f1e2..ce8fe4c 100644
--- a/hadoop-ozone/objectstore-service/pom.xml
+++ b/hadoop-ozone/objectstore-service/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Object Store REST Service</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index 3efb33b..a3f69f6 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -28,11 +28,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Manager Server</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
@@ -63,28 +58,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>copy web resources</id>
- <phase>compile</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <copy toDir="${project.build.directory}/webapps">
- <fileset dir="${basedir}/src/main/webapps">
- <exclude name="**/proto-web.xml"/>
- </fileset>
- </copy>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
@@ -93,19 +66,20 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals>
<goal>unpack</goal>
</goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdds-server-framework</artifactId>
+ <outputDirectory>${project.build.outputDirectory}
+ </outputDirectory>
+ <includes>webapps/static/**/*.*</includes>
+ </artifactItem>
+ </artifactItems>
+ <overWriteSnapshots>true</overWriteSnapshots>
+ </configuration>
</execution>
</executions>
- <configuration>
- <artifactItems>
- <artifactItem>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdds-server-framework</artifactId>
- <outputDirectory>${project.build.directory}/</outputDirectory>
- <includes>webapps/static/**/*.*</includes>
- </artifactItem>
- </artifactItems>
- <overWriteSnapshots>true</overWriteSnapshots>
- </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
new file mode 100644
index 0000000..ba54cb2
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html
@@ -0,0 +1,70 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+ <meta name="description" content="Ozone Manager">
+
+ <title>Ozone Manager</title>
+
+ <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
+ <link href="static/hadoop.css" rel="stylesheet">
+ <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
+
+ <link href="static/ozone.css" rel="stylesheet">
+
+</head>
+
+<body ng-app="ozoneManager">
+
+<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
+ <div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
+ aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="#">Ozone Manager</a>
+ </div>
+ <navmenu
+ metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
+ </div>
+</header>
+
+<div class="container-fluid">
+ <ng-view></ng-view>
+</div><!-- /.container -->
+
+<script src="static/jquery-3.3.1.min.js"></script>
+<script src="static/angular-1.6.4.min.js"></script>
+<script src="static/angular-route-1.6.4.min.js"></script>
+<script src="static/d3-3.5.17.min.js"></script>
+<script src="static/nvd3-1.8.5.min.js"></script>
+<script src="static/angular-nvd3-1.0.9.min.js"></script>
+<script src="static/ozone.js"></script>
+<script src="ozoneManager.js"></script>
+<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
new file mode 100644
index 0000000..e442adc
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.css
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+body {
+ padding-top: 50px;
+}
+.starter-template {
+ padding: 40px 15px;
+ text-align: center;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
new file mode 100644
index 0000000..0821899
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/main.html
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<overview>
+</overview>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
new file mode 100644
index 0000000..15fba2f
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-metrics.html
@@ -0,0 +1,44 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<h1>OzoneManager Metrics</h1>
+
+<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
+ <h2>{{type}}</h2>
+ <div class="container">
+ <div class="col-md-6">
+ <h3>Requests ({{numbers.ops}} ops)</h3>
+ <nvd3 options="$ctrl.graphOptions"
+ data="numbers.all"></nvd3>
+ </div>
+ <div class="col-md-6">
+ <h3>Failures</h3>
+ <nvd3 options="$ctrl.graphOptions"
+ data="numbers.failures"></nvd3>
+ </div>
+ </div>
+</div>
+
+<div ng-show="$ctrl.metrics.others.length > 0">
+ <h2>Other JMX properties</h2>
+
+ <table class="table">
+ <tr ng-repeat="metric in $ctrl.metrics.others">
+ <td>{{metric.key}}</td>
+ <td>{{metric.value}}</td>
+ </tr>
+ </table>
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
new file mode 100644
index 0000000..ca03554
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ "use strict";
+
+ var isIgnoredJmxKeys = function (key) {
+ return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
+ };
+
+ angular.module('ozoneManager', ['ozone', 'nvd3']);
+ angular.module('ozoneManager').config(function ($routeProvider) {
+ $routeProvider
+ .when("/metrics/ozoneManager", {
+ template: "<om-metrics></om-metrics>"
+ });
+ });
+ angular.module('ozoneManager').component('omMetrics', {
+ templateUrl: 'om-metrics.html',
+ controller: function ($http) {
+ var ctrl = this;
+
+ ctrl.graphOptions = {
+ chart: {
+ type: 'pieChart',
+ height: 500,
+ x: function (d) {
+ return d.key;
+ },
+ y: function (d) {
+ return d.value;
+ },
+ showLabels: true,
+ labelType: 'value',
+ duration: 500,
+ labelThreshold: 0.01,
+ valueFormat: function(d) {
+ return d3.format('d')(d);
+ },
+ legend: {
+ margin: {
+ top: 5,
+ right: 35,
+ bottom: 5,
+ left: 0
+ }
+ }
+ }
+ };
+
+
+ $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
+ .then(function (result) {
+
+ var groupedMetrics = {others: [], nums: {}};
+ var metrics = result.data.beans[0]
+ for (var key in metrics) {
+ var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
+ if (numericalStatistic) {
+ var type = numericalStatistic[1];
+ var name = numericalStatistic[2];
+ var failed = numericalStatistic[3];
+ groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
+ failures: [],
+ all: []
+ };
+ if (failed) {
+ groupedMetrics.nums[type].failures.push({
+ key: name,
+ value: metrics[key]
+ })
+ } else {
+ if (name == "Ops") {
+ groupedMetrics.nums[type].ops = metrics[key]
+ } else {
+ groupedMetrics.nums[type].all.push({
+ key: name,
+ value: metrics[key]
+ })
+ }
+ }
+ } else if (isIgnoredJmxKeys(key)) {
+ //ignore
+ } else {
+ groupedMetrics.others.push({
+ 'key': key,
+ 'value': metrics[key]
+ });
+ }
+ }
+ ctrl.metrics = groupedMetrics;
+ })
+ }
+ });
+
+})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
deleted file mode 100644
index ba54cb2..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/index.html
+++ /dev/null
@@ -1,70 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <meta name="description" content="Ozone Manager">
-
- <title>Ozone Manager</title>
-
- <link href="static/bootstrap-3.3.7/css/bootstrap.min.css" rel="stylesheet">
- <link href="static/hadoop.css" rel="stylesheet">
- <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
- <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="ozoneManager">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
- aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="#">Ozone Manager</a>
- </div>
- <navmenu
- metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
- </div>
-</header>
-
-<div class="container-fluid">
- <ng-view></ng-view>
-</div><!-- /.container -->
-
-<script src="static/jquery-3.3.1.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="ozoneManager.js"></script>
-<script src="static/bootstrap-3.3.7/js/bootstrap.min.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
deleted file mode 100644
index e442adc..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.css
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-body {
- padding-top: 50px;
-}
-.starter-template {
- padding: 40px 15px;
- text-align: center;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
deleted file mode 100644
index 0821899..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/main.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<overview>
-</overview>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
deleted file mode 100644
index 15fba2f..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/om-metrics.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h1>OzoneManager Metrics</h1>
-
-<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
- <h2>{{type}}</h2>
- <div class="container">
- <div class="col-md-6">
- <h3>Requests ({{numbers.ops}} ops)</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.all"></nvd3>
- </div>
- <div class="col-md-6">
- <h3>Failures</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.failures"></nvd3>
- </div>
- </div>
-</div>
-
-<div ng-show="$ctrl.metrics.others.length > 0">
- <h2>Other JMX properties</h2>
-
- <table class="table">
- <tr ng-repeat="metric in $ctrl.metrics.others">
- <td>{{metric.key}}</td>
- <td>{{metric.value}}</td>
- </tr>
- </table>
-</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
deleted file mode 100644
index ca03554..0000000
--- a/hadoop-ozone/ozone-manager/src/main/webapps/ozoneManager/ozoneManager.js
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- var isIgnoredJmxKeys = function (key) {
- return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
- };
-
- angular.module('ozoneManager', ['ozone', 'nvd3']);
- angular.module('ozoneManager').config(function ($routeProvider) {
- $routeProvider
- .when("/metrics/ozoneManager", {
- template: "<om-metrics></om-metrics>"
- });
- });
- angular.module('ozoneManager').component('omMetrics', {
- templateUrl: 'om-metrics.html',
- controller: function ($http) {
- var ctrl = this;
-
- ctrl.graphOptions = {
- chart: {
- type: 'pieChart',
- height: 500,
- x: function (d) {
- return d.key;
- },
- y: function (d) {
- return d.value;
- },
- showLabels: true,
- labelType: 'value',
- duration: 500,
- labelThreshold: 0.01,
- valueFormat: function(d) {
- return d3.format('d')(d);
- },
- legend: {
- margin: {
- top: 5,
- right: 35,
- bottom: 5,
- left: 0
- }
- }
- }
- };
-
-
- $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics")
- .then(function (result) {
-
- var groupedMetrics = {others: [], nums: {}};
- var metrics = result.data.beans[0]
- for (var key in metrics) {
- var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
- if (numericalStatistic) {
- var type = numericalStatistic[1];
- var name = numericalStatistic[2];
- var failed = numericalStatistic[3];
- groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
- failures: [],
- all: []
- };
- if (failed) {
- groupedMetrics.nums[type].failures.push({
- key: name,
- value: metrics[key]
- })
- } else {
- if (name == "Ops") {
- groupedMetrics.nums[type].ops = metrics[key]
- } else {
- groupedMetrics.nums[type].all.push({
- key: name,
- value: metrics[key]
- })
- }
- }
- } else if (isIgnoredJmxKeys(key)) {
- //ignore
- } else {
- groupedMetrics.others.push({
- 'key': key,
- 'value': metrics[key]
- });
- }
- }
- ctrl.metrics = groupedMetrics;
- })
- }
- });
-
-})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index c45aacd..8174b74 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -28,8 +28,6 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
</properties>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index c73be60..8d3e161 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -19,9 +19,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-project-dist</artifactId>
+ <artifactId>hadoop-project</artifactId>
<version>3.2.0-SNAPSHOT</version>
- <relativePath>../hadoop-project-dist</relativePath>
+ <relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-ozone</artifactId>
<version>0.3.0-SNAPSHOT</version>
@@ -42,6 +42,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<module>ozonefs</module>
<module>integration-test</module>
<module>objectstore-service</module>
+ <module>datanode</module>
+ <module>dist</module>
<module>docs</module>
</modules>
@@ -161,6 +163,25 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<excludeFilterFile combine.self="override"></excludeFilterFile>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-classpath-descriptor</id>
+ <phase>package</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <attach>true</attach>
+ <prefix>$HDDS_LIB_JARS_DIR</prefix>
+ <outputFilterFile>true</outputFilterFile>
+ <includeScope>runtime</includeScope>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index eeec595..b04bbf1 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -28,21 +28,33 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Tools</name>
<packaging>jar</packaging>
- <properties>
- <hadoop.component>ozone</hadoop.component>
- <is.hadoop.component>true</is.hadoop.component>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-client</artifactId>
- <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-filesystem</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
@@ -59,6 +71,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>jmh-generator-annprocess</artifactId>
<version>1.19</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-integration-test</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
new file mode 100644
index 0000000..a2df50d
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Freon, with MiniOzoneCluster and validate data.
+ */
+public class TestDataValidate {
+
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(5).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void ratisTestLargeKey() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setKeySize(20971520);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
+ }
+
+ @Test
+ public void standaloneTestLargeKey() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(1);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(1);
+ randomKeyGenerator.setKeySize(20971520);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
+ }
+
+ @Test
+ public void validateWriteTest() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(2);
+ randomKeyGenerator.setNumOfBuckets(5);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setValidateWrites(true);
+ randomKeyGenerator.call();
+ Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertTrue(randomKeyGenerator.getValidateWrites());
+ Assert.assertNotEquals(0, randomKeyGenerator.getTotalKeysValidated());
+ Assert.assertNotEquals(0, randomKeyGenerator
+ .getSuccessfulValidationCount());
+ Assert.assertEquals(0, randomKeyGenerator
+ .getUnsuccessfulValidationCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
new file mode 100644
index 0000000..d21d399
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Freon, with MiniOzoneCluster.
+ */
+public class TestRandomKeyGenerator {
+
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void defaultTest() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(2);
+ randomKeyGenerator.setNumOfBuckets(5);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.call();
+ Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ Assert.assertEquals(10240 - 36, randomKeyGenerator.getKeyValueLength());
+ }
+
+ @Test
+ public void multiThread() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(10);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setNumOfThreads(10);
+ randomKeyGenerator.setKeySize(10240);
+ randomKeyGenerator.call();
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ }
+
+ @Test
+ public void ratisTest3() throws Exception {
+ RandomKeyGenerator randomKeyGenerator =
+ new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
+ randomKeyGenerator.setNumOfVolumes(10);
+ randomKeyGenerator.setNumOfBuckets(1);
+ randomKeyGenerator.setNumOfKeys(10);
+ randomKeyGenerator.setNumOfThreads(10);
+ randomKeyGenerator.setKeySize(10240);
+ randomKeyGenerator.setFactor(ReplicationFactor.THREE);
+ randomKeyGenerator.setType(ReplicationType.RATIS);
+ randomKeyGenerator.call();
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
+ Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
+ Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
new file mode 100644
index 0000000..a3ff6c8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.scm.cli.SQLCLI;
+import org.apache.hadoop.ozone.web.handlers.BucketArgs;
+import org.apache.hadoop.ozone.web.handlers.KeyArgs;
+import org.apache.hadoop.ozone.web.handlers.UserArgs;
+import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
+import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the CLI that transforms om.db into SQLite DB files.
+ */
+@RunWith(Parameterized.class)
+public class TestOmSQLCli {
+ private MiniOzoneCluster cluster = null;
+ private StorageHandler storageHandler;
+ private UserArgs userArgs;
+ private OzoneConfiguration conf;
+ private SQLCLI cli;
+
+ private String userName = "userTest";
+ private String adminName = "adminTest";
+ private String volumeName0 = "volumeTest0";
+ private String volumeName1 = "volumeTest1";
+ private String bucketName0 = "bucketTest0";
+ private String bucketName1 = "bucketTest1";
+ private String bucketName2 = "bucketTest2";
+ private String keyName0 = "key0";
+ private String keyName1 = "key1";
+ private String keyName2 = "key2";
+ private String keyName3 = "key3";
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ // Uncomment the below line if we support leveldb in future.
+ //{OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private String metaStoreType;
+
+ public TestOmSQLCli(String type) {
+ metaStoreType = type;
+ }
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true
+ *
+ * @throws IOException
+ */
+ @Before
+ public void setup() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf).build();
+ cluster.waitForClusterToBeReady();
+ storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
+ userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
+ null, null, null, null);
+ cluster.waitForClusterToBeReady();
+
+ VolumeArgs createVolumeArgs0 = new VolumeArgs(volumeName0, userArgs);
+ createVolumeArgs0.setUserName(userName);
+ createVolumeArgs0.setAdminName(adminName);
+ storageHandler.createVolume(createVolumeArgs0);
+ VolumeArgs createVolumeArgs1 = new VolumeArgs(volumeName1, userArgs);
+ createVolumeArgs1.setUserName(userName);
+ createVolumeArgs1.setAdminName(adminName);
+ storageHandler.createVolume(createVolumeArgs1);
+
+ BucketArgs bucketArgs0 = new BucketArgs(volumeName0, bucketName0, userArgs);
+ storageHandler.createBucket(bucketArgs0);
+ BucketArgs bucketArgs1 = new BucketArgs(volumeName1, bucketName1, userArgs);
+ storageHandler.createBucket(bucketArgs1);
+ BucketArgs bucketArgs2 = new BucketArgs(volumeName0, bucketName2, userArgs);
+ storageHandler.createBucket(bucketArgs2);
+
+ KeyArgs keyArgs0 =
+ new KeyArgs(volumeName0, bucketName0, keyName0, userArgs);
+ keyArgs0.setSize(100);
+ KeyArgs keyArgs1 =
+ new KeyArgs(volumeName1, bucketName1, keyName1, userArgs);
+ keyArgs1.setSize(200);
+ KeyArgs keyArgs2 =
+ new KeyArgs(volumeName0, bucketName2, keyName2, userArgs);
+ keyArgs2.setSize(300);
+ KeyArgs keyArgs3 =
+ new KeyArgs(volumeName0, bucketName2, keyName3, userArgs);
+ keyArgs3.setSize(400);
+
+ OutputStream stream = storageHandler.newKeyWriter(keyArgs0);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs1);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs2);
+ stream.close();
+ stream = storageHandler.newKeyWriter(keyArgs3);
+ stream.close();
+
+ cluster.getOzoneManager().stop();
+ cluster.getStorageContainerManager().stop();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
+ cli = new SQLCLI(conf);
+ }
+
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ // After HDDS-357, we have to fix SQLCli.
+ // TODO: fix SQLCli
+ @Ignore
+ @Test
+ public void testOmDB() throws Exception {
+ String dbOutPath = GenericTestUtils.getTempPath(
+ UUID.randomUUID() + "/out_sql.db");
+
+ String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbPath = dbRootPath + "/" + OM_DB_NAME;
+ String[] args = {"-p", dbPath, "-o", dbOutPath};
+
+ cli.run(args);
+
+ Connection conn = connectDB(dbOutPath);
+ String sql = "SELECT * FROM volumeList";
+ ResultSet rs = executeQuery(conn, sql);
+ List<String> expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String userNameRs = rs.getString("userName");
+ String volumeNameRs = rs.getString("volumeName");
+ assertEquals(userName, userNameRs.substring(1));
+ assertTrue(expectedValues.remove(volumeNameRs));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM volumeInfo";
+ rs = executeQuery(conn, sql);
+ expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String adName = rs.getString("adminName");
+ String ownerName = rs.getString("ownerName");
+ String volumeName = rs.getString("volumeName");
+ assertEquals(adminName, adName);
+ assertEquals(userName, ownerName);
+ assertTrue(expectedValues.remove(volumeName));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM aclInfo";
+ rs = executeQuery(conn, sql);
+ expectedValues =
+ new LinkedList<>(Arrays.asList(volumeName0, volumeName1));
+ while (rs.next()) {
+ String adName = rs.getString("adminName");
+ String ownerName = rs.getString("ownerName");
+ String volumeName = rs.getString("volumeName");
+ String type = rs.getString("type");
+ String uName = rs.getString("userName");
+ String rights = rs.getString("rights");
+ assertEquals(adminName, adName);
+ assertEquals(userName, ownerName);
+ assertEquals("USER", type);
+ assertEquals(userName, uName);
+ assertEquals("READ_WRITE", rights);
+ assertTrue(expectedValues.remove(volumeName));
+ }
+ assertEquals(0, expectedValues.size());
+
+ sql = "SELECT * FROM bucketInfo";
+ rs = executeQuery(conn, sql);
+ HashMap<String, String> expectedMap = new HashMap<>();
+ expectedMap.put(bucketName0, volumeName0);
+ expectedMap.put(bucketName2, volumeName0);
+ expectedMap.put(bucketName1, volumeName1);
+ while (rs.next()) {
+ String volumeName = rs.getString("volumeName");
+ String bucketName = rs.getString("bucketName");
+ boolean versionEnabled = rs.getBoolean("versionEnabled");
+ String storegeType = rs.getString("storageType");
+ assertEquals(volumeName, expectedMap.remove(bucketName));
+ assertFalse(versionEnabled);
+ assertEquals("DISK", storegeType);
+ }
+ assertEquals(0, expectedMap.size());
+
+ sql = "SELECT * FROM keyInfo";
+ rs = executeQuery(conn, sql);
+ HashMap<String, List<String>> expectedMap2 = new HashMap<>();
+ // no data written, data size will be 0
+ expectedMap2.put(keyName0,
+ Arrays.asList(volumeName0, bucketName0, "0"));
+ expectedMap2.put(keyName1,
+ Arrays.asList(volumeName1, bucketName1, "0"));
+ expectedMap2.put(keyName2,
+ Arrays.asList(volumeName0, bucketName2, "0"));
+ expectedMap2.put(keyName3,
+ Arrays.asList(volumeName0, bucketName2, "0"));
+ while (rs.next()) {
+ String volumeName = rs.getString("volumeName");
+ String bucketName = rs.getString("bucketName");
+ String keyName = rs.getString("keyName");
+ int dataSize = rs.getInt("dataSize");
+ List<String> vals = expectedMap2.remove(keyName);
+ assertNotNull(vals);
+ assertEquals(vals.get(0), volumeName);
+ assertEquals(vals.get(1), bucketName);
+ assertEquals(vals.get(2), Integer.toString(dataSize));
+ }
+ assertEquals(0, expectedMap2.size());
+
+ conn.close();
+ Files.delete(Paths.get(dbOutPath));
+ }
+
+ private ResultSet executeQuery(Connection conn, String sql)
+ throws SQLException {
+ Statement stmt = conn.createStatement();
+ return stmt.executeQuery(sql);
+ }
+
+ private Connection connectDB(String dbPath) throws Exception {
+ Class.forName("org.sqlite.JDBC");
+ String connectPath =
+ String.format("jdbc:sqlite:%s", dbPath);
+ return DriverManager.getConnection(connectPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
new file mode 100644
index 0000000..4026348
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -0,0 +1,253 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright containerOwnership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm;
+
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
+import org.apache.hadoop.hdds.scm.container.ContainerMapping;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
+import org.apache.hadoop.ozone.scm.cli.SQLCLI;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
+import static org.apache.hadoop.ozone.OzoneConsts.KB;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class tests the CLI that transforms container into SQLite DB files.
+ */
+@RunWith(Parameterized.class)
+public class TestContainerSQLCli {
+
+ private EventQueue eventQueue;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB},
+ {OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB}
+ });
+ }
+
+ private static String metaStoreType;
+
+ public TestContainerSQLCli(String type) {
+ metaStoreType = type;
+ }
+
+ private static SQLCLI cli;
+
+ private MiniOzoneCluster cluster;
+ private OzoneConfiguration conf;
+ private String datanodeIpAddress;
+
+ private ContainerMapping mapping;
+ private NodeManager nodeManager;
+ private BlockManagerImpl blockManager;
+
+ private HashMap<Long, Long> blockContainerMap;
+
+ private final static long DEFAULT_BLOCK_SIZE = 4 * KB;
+ private static HddsProtos.ReplicationFactor factor;
+ private static HddsProtos.ReplicationType type;
+ private static final String CONTAINER_OWNER = "OZONE";
+
+
+ @Before
+ public void setup() throws Exception {
+ blockContainerMap = new HashMap<>();
+
+ conf = new OzoneConfiguration();
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 2);
+ conf.setClass(ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
+ SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
+ if(conf.getBoolean(ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
+ ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT)){
+ factor = HddsProtos.ReplicationFactor.THREE;
+ type = HddsProtos.ReplicationType.RATIS;
+ } else {
+ factor = HddsProtos.ReplicationFactor.ONE;
+ type = HddsProtos.ReplicationType.STAND_ALONE;
+ }
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2).build();
+ cluster.waitForClusterToBeReady();
+ datanodeIpAddress = cluster.getHddsDatanodes().get(0)
+ .getDatanodeDetails().getIpAddress();
+ cluster.getOzoneManager().stop();
+ cluster.getStorageContainerManager().stop();
+ eventQueue = new EventQueue();
+ nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
+ mapping = new ContainerMapping(conf, nodeManager, 128,
+ eventQueue);
+ blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
+ eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
+ GenericTestUtils.waitFor(() -> {
+ return !blockManager.isScmInChillMode();
+ }, 10, 1000 * 15);
+ // blockManager.allocateBlock() will create containers if there is none
+ // stored in levelDB. The number of containers to create is the value of
+ // OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2.
+ // so the first allocateBlock() will create two containers. A random one
+ // is assigned for the block.
+
+ // loop until both the two datanodes are up, try up to about 4 seconds.
+ for (int c = 0; c < 40; c++) {
+ if (nodeManager.getAllNodes().size() == 2) {
+ break;
+ }
+ Thread.sleep(100);
+ }
+ assertEquals(2, nodeManager.getAllNodes().size());
+ AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type,
+ factor, CONTAINER_OWNER);
+ blockContainerMap.put(ab1.getBlockID().getLocalID(),
+ ab1.getBlockID().getContainerID());
+
+ AllocatedBlock ab2;
+ // we want the two blocks on the two provisioned containers respectively,
+ // however blockManager picks containers randomly, keep retry until we
+ // assign the second block to the other container. This seems to be the only
+ // way to get the two containers.
+ // although each retry will create a block and assign to a container. So
+ // the size of blockContainerMap will vary each time the test is run.
+ while (true) {
+ ab2 = blockManager
+ .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER);
+ blockContainerMap.put(ab2.getBlockID().getLocalID(),
+ ab2.getBlockID().getContainerID());
+ if (ab1.getBlockID().getContainerID() !=
+ ab2.getBlockID().getContainerID()) {
+ break;
+ }
+ }
+
+ blockManager.close();
+ mapping.close();
+ nodeManager.close();
+
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
+ cli = new SQLCLI(conf);
+
+ }
+
+ @After
+ public void shutdown() throws InterruptedException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testConvertContainerDB() throws Exception {
+ String dbOutPath = GenericTestUtils.getTempPath(
+ UUID.randomUUID() + "/out_sql.db");
+ // TODO : the following will fail due to empty Datanode list, need to fix.
+ //String dnUUID = cluster.getDataNodes().get(0).getUuid();
+ String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
+ String dbPath = dbRootPath + "/" + SCM_CONTAINER_DB;
+ String[] args = {"-p", dbPath, "-o", dbOutPath};
+ Connection conn;
+ String sql;
+ ResultSet rs;
+
+ cli.run(args);
+
+ //verify the sqlite db
+ // only checks the container names are as expected. Because other fields
+ // such as datanode UUID are generated randomly each time
+ conn = connectDB(dbOutPath);
+ sql = "SELECT * FROM containerInfo";
+ rs = executeQuery(conn, sql);
+ ArrayList<Long> containerIDs = new ArrayList<>();
+ while (rs.next()) {
+ containerIDs.add(rs.getLong("containerID"));
+ //assertEquals(dnUUID, rs.getString("leaderUUID"));
+ }
+ /* TODO: fix this later when the SQLCLI is fixed.
+ assertTrue(containerIDs.size() == 2 &&
+ containerIDs.contains(pipeline1.getContainerName()) &&
+ containerIDs.contains(pipeline2.getContainerName()));
+
+ sql = "SELECT * FROM containerMembers";
+ rs = executeQuery(conn, sql);
+ containerIDs = new ArrayList<>();
+ while (rs.next()) {
+ containerIDs.add(rs.getLong("containerID"));
+ //assertEquals(dnUUID, rs.getString("datanodeUUID"));
+ }
+ assertTrue(containerIDs.size() == 2 &&
+ containerIDs.contains(pipeline1.getContainerName()) &&
+ containerIDs.contains(pipeline2.getContainerName()));
+
+ sql = "SELECT * FROM datanodeInfo";
+ rs = executeQuery(conn, sql);
+ int count = 0;
+ while (rs.next()) {
+ assertEquals(datanodeIpAddress, rs.getString("ipAddress"));
+ //assertEquals(dnUUID, rs.getString("datanodeUUID"));
+ count += 1;
+ }
+ // the two containers maybe on the same datanode, maybe not.
+ int expected = pipeline1.getLeader().getUuid().equals(
+ pipeline2.getLeader().getUuid())? 1 : 2;
+ assertEquals(expected, count);
+ */
+ Files.delete(Paths.get(dbOutPath));
+ }
+
+ private ResultSet executeQuery(Connection conn, String sql)
+ throws SQLException {
+ Statement stmt = conn.createStatement();
+ return stmt.executeQuery(sql);
+ }
+
+ private Connection connectDB(String dbPath) throws Exception {
+ Class.forName("org.sqlite.JDBC");
+ String connectPath =
+ String.format("jdbc:sqlite:%s", dbPath);
+ return DriverManager.getConnection(connectPath);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62f817d3/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 794fdcc..8d08703 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -618,6 +618,11 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-datanode</artifactId>
+ <version>${hdds.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-integration-test</artifactId>
<version>${hdds.version}</version>
<type>test-jar</type>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org