You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2018/10/29 14:17:31 UTC
[1/6] hadoop git commit: HDDS-714. Bump protobuf version to 3.5.1.
Contributed by Mukul Kumar Singh.
Repository: hadoop
Updated Branches:
refs/heads/ozone-0.3 095e8f61b -> 031cdf5d5
HDDS-714. Bump protobuf version to 3.5.1. Contributed by Mukul Kumar Singh.
(cherry picked from commit ace06a93baa09293c254d18c709162771738b092)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/031cdf5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/031cdf5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/031cdf5d
Branch: refs/heads/ozone-0.3
Commit: 031cdf5d5f9745516ef1f2b83b91c5ccd45ae736
Parents: 4c71caa
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Thu Oct 25 09:34:06 2018 +0530
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031cdf5d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 22c3b35..ec018a6 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -110,7 +110,7 @@
<!-- Maven protoc compiler -->
<protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
- <protobuf-compile.version>3.5.0</protobuf-compile.version>
+ <protobuf-compile.version>3.5.1</protobuf-compile.version>
<grpc.version>1.10.0</grpc.version>
<os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[6/6] hadoop git commit: HDDS-568. ozone sh volume info, update,
delete operations fail when volume name is not prefixed by /.
Contributed by Dinesh Chitlangia.
Posted by el...@apache.org.
HDDS-568. ozone sh volume info, update, delete operations fail when volume name is not prefixed by /.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0d0ef58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0d0ef58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0d0ef58
Branch: refs/heads/ozone-0.3
Commit: e0d0ef58db4398903f4962099e4eb2b1b677e5f0
Parents: 911b570
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:16:52 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
.../hadoop/ozone/ozShell/TestOzoneShell.java | 48 ++++++++++++++++++++
.../hadoop/ozone/web/ozShell/Handler.java | 28 ++++++++++++
.../web/ozShell/volume/DeleteVolumeHandler.java | 12 +----
.../web/ozShell/volume/InfoVolumeHandler.java | 22 +--------
.../web/ozShell/volume/UpdateVolumeHandler.java | 12 +----
5 files changed, 79 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index c736f81..a7eecc0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -305,6 +305,33 @@ public class TestOzoneShell {
GenericTestUtils.assertExceptionContains(
"Info Volume failed, error:VOLUME_NOT_FOUND", e);
}
+
+
+ volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ volumeArgs = VolumeArgs.newBuilder()
+ .setOwner("bilbo")
+ .setQuota("100TB")
+ .build();
+ client.createVolume(volumeName, volumeArgs);
+ volume = client.getVolumeDetails(volumeName);
+ assertNotNull(volume);
+
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "delete",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+ output = out.toString();
+ assertTrue(output.contains("Volume " + volumeName + " is deleted"));
+
+ // verify if volume has been deleted
+ try {
+ client.getVolumeDetails(volumeName);
+ fail("Get volume call should have thrown.");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Info Volume failed, error:VOLUME_NOT_FOUND", e);
+ }
}
@Test
@@ -317,6 +344,7 @@ public class TestOzoneShell {
.build();
client.createVolume(volumeName, volumeArgs);
+ //volumeName supplied as-is
String[] args = new String[] {"volume", "info", url + "/" + volumeName};
execute(shell, args);
@@ -325,6 +353,17 @@ public class TestOzoneShell {
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
+ //volumeName prefixed with /
+ String volumeNameWithSlashPrefix = "/" + volumeName;
+ args = new String[] {"volume", "info",
+ url + "/" + volumeNameWithSlashPrefix};
+ execute(shell, args);
+
+ output = out.toString();
+ assertTrue(output.contains(volumeName));
+ assertTrue(output.contains("createdOn")
+ && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+
// test infoVolume with invalid volume name
args = new String[] {"volume", "info",
url + "/" + volumeName + "/invalid-name"};
@@ -387,6 +426,15 @@ public class TestOzoneShell {
vol = client.getVolumeDetails(volumeName);
assertEquals(newUser, vol.getOwner());
+ //volume with / prefix
+ String volumeWithPrefix = "/" + volumeName;
+ String newUser2 = "new-user2";
+ args = new String[] {"volume", "update", url + "/" + volumeWithPrefix,
+ "--user", newUser2};
+ execute(shell, args);
+ vol = client.getVolumeDetails(volumeName);
+ assertEquals(newUser2, vol.getOwner());
+
// test error conditions
args = new String[] {"volume", "update", url + "/invalid-volume",
"--user", newUser};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index 5067855..be579b3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.web.ozShell;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.concurrent.Callable;
import org.apache.hadoop.conf.Configuration;
@@ -153,6 +155,32 @@ public abstract class Handler implements Callable<Void> {
}
}
+ /**
+ *
+ * @param uri
+ * @return volumeName
+ * @throws Exception
+ * @throws OzoneClientException when uri is null or invalid volume name
+ */
+ protected String parseVolumeName(String uri) throws Exception{
+ URI ozoneURI = verifyURI(uri);
+ Path path = Paths.get(ozoneURI.getPath());
+ int pathNameCount = path.getNameCount();
+ if (pathNameCount != 1) {
+ String errorMessage;
+ if (pathNameCount < 1) {
+ errorMessage = "Volume name is required to perform volume " +
+ "operations like info, update, create and delete. ";
+ } else {
+ errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+ "volume name";
+ }
+ throw new OzoneClientException(errorMessage);
+ }
+
+ return ozoneURI.getPath().replaceAll("^/+", "");
+ }
+
public boolean isVerbose() {
return parent.isVerbose();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
index d1e96fc..d757e20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
@@ -43,14 +40,7 @@ public class DeleteVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to delete a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
if (isVerbose()) {
System.out.printf("Volume name : %s%n", volumeName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
index 16c0342..48ed9f6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
@@ -18,11 +18,6 @@
package org.apache.hadoop.ozone.web.ozShell.volume;
-import java.net.URI;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.web.ozShell.Handler;
@@ -48,22 +43,7 @@ public class InfoVolumeHandler extends Handler{
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- Path path = Paths.get(ozoneURI.getPath());
- int pathNameCount = path.getNameCount();
- if (pathNameCount != 1) {
- String errorMessage;
- if (pathNameCount < 1) {
- errorMessage = "Volume name is required to get info of a volume";
- } else {
- errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
- "volume name";
- }
- throw new OzoneClientException(errorMessage);
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
index 0336fc2..46803d8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
@@ -25,13 +25,10 @@ import picocli.CommandLine.Parameters;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneClientException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import java.net.URI;
-
/**
* Executes update volume calls.
*/
@@ -57,14 +54,7 @@ public class UpdateVolumeHandler extends Handler {
@Override
public Void call() throws Exception {
- URI ozoneURI = verifyURI(uri);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneClientException(
- "Volume name is required to update a volume");
- }
-
- // we need to skip the slash in the URI path
- String volumeName = ozoneURI.getPath().substring(1);
+ String volumeName = parseVolumeName(uri);
OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
if (quota != null && !quota.isEmpty()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/6] hadoop git commit: HDDS-443. Create reusable ProgressBar
utility for freon tests. Contributed by Zsolt Horvath.
Posted by el...@apache.org.
HDDS-443. Create reusable ProgressBar utility for freon tests.
Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c71caa3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c71caa3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c71caa3
Branch: refs/heads/ozone-0.3
Commit: 4c71caa30b8eabb2aedad8b619eb16711547ab91
Parents: 5bec17e
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 18:18:19 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
hadoop-ozone/tools/pom.xml | 6 +
.../apache/hadoop/ozone/freon/ProgressBar.java | 210 +++++++++++++++++++
.../hadoop/ozone/freon/TestProgressBar.java | 112 ++++++++++
3 files changed, 328 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index b04bbf1..560e072 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -77,6 +77,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.15.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
new file mode 100644
index 0000000..a8d7e73
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
+ */
+public class ProgressBar {
+
+ private static final long REFRESH_INTERVAL = 1000L;
+
+ private PrintStream stream;
+ private AtomicLong currentValue;
+ private long maxValue;
+ private Thread progressBar;
+ private volatile boolean exception = false;
+ private long startTime;
+
+ /**
+ * @param stream Used to display the progress
+ * @param maxValue Maximum value of the progress
+ */
+ ProgressBar(PrintStream stream, long maxValue) {
+ this.stream = stream;
+ this.maxValue = maxValue;
+ this.currentValue = new AtomicLong(0);
+ this.progressBar = new Thread(new ProgressBarThread());
+ }
+
+ /**
+ * Start a task with a progessbar without any in/out parameters Runnable used
+ * just a task wrapper.
+ *
+ * @param task Runnable
+ */
+ public void start(Runnable task) {
+
+ startTime = System.nanoTime();
+
+ try {
+
+ progressBar.start();
+ task.run();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Start a task with only out parameters.
+ *
+ * @param task Supplier that represents the task
+ * @param <T> Generic return type
+ * @return Whatever the supllier produces
+ */
+ public <T> T start(Supplier<T> task) {
+
+ startTime = System.nanoTime();
+ T result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.get();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Start a task with in/out parameters.
+ *
+ * @param input Input of the function
+ * @param task A Function that does the task
+ * @param <T> type of the input
+ * @param <R> return type
+ * @return Whatever the Function returns
+ */
+ public <T, R> R start(T input, Function<T, R> task) {
+
+ startTime = System.nanoTime();
+ R result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.apply(input);
+
+ } catch (Exception e) {
+ exception = true;
+ throw e;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return result;
+ }
+ }
+
+ /**
+ * Increment the progress with one step.
+ */
+ public void incrementProgress() {
+ currentValue.incrementAndGet();
+ }
+
+ private class ProgressBarThread implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+
+ stream.println();
+ long value;
+
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+
+ if (exception) {
+ break;
+ }
+ Thread.sleep(REFRESH_INTERVAL);
+ }
+
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " + "check log for " +
+ "exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ stream.println(e);
+ }
+ }
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
new file mode 100644
index 0000000..cea7be8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.PrintStream;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+/**
+ * Tests for the Progressbar class for Freon.
+ */
+public class TestProgressBar {
+
+ private PrintStream stream;
+
+ @Before
+ public void setupMock() {
+ stream = mock(PrintStream.class);
+ }
+
+ @Test
+ public void testWithRunnable() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Runnable task = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+ };
+
+ progressbar.start(task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithSupplier() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Supplier<Long> tasks = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return 1L; //return the result of the dummy task
+ };
+
+ progressbar.start(tasks);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithFunction() {
+
+ int maxValue = 10;
+ Long result;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Function<Long, String> task = (Long l) -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return "dummy result"; //return the result of the dummy task
+ };
+
+ progressbar.start(1L, task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/6] hadoop git commit: HDDS-478. Log files related to each daemon
doesn't have proper startup and shutdown logs. Contributed by Dinesh
Chitlangia.
Posted by el...@apache.org.
HDDS-478. Log files related to each daemon doesn't have proper startup and shutdown logs.
Contributed by Dinesh Chitlangia.
(cherry picked from commit c1fe657a106aaae3bdf81fa4add70962aaee165b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78ec30cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78ec30cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78ec30cf
Branch: refs/heads/ozone-0.3
Commit: 78ec30cf4e6ce7078ecf8144dece0c34c40150ea
Parents: 095e8f6
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:44:32 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
.../common/src/main/conf/om-audit-log4j2.properties | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ec30cf/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
index 7d097a0..7be51ac 100644
--- a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
+++ b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
@@ -52,11 +52,15 @@ filter.write.onMismatch=NEUTRAL
# TRACE (least specific, a lot of data)
# ALL (least specific, all data)
-appenders=console, rolling
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+# Uncomment following section to enable logging to console appender also
+#appenders=console, rolling
+#appender.console.type=Console
+#appender.console.name=STDOUT
+#appender.console.layout.type=PatternLayout
+#appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+
+# Comment this line when using both console and rolling appenders
+appenders=rolling
#Rolling File Appender with size & time thresholds.
#Rolling is triggered when either threshold is breached.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/6] hadoop git commit: HDDS-559. fs.default.name is deprecated.
Contributed by Dinesh Chitlangia.
Posted by el...@apache.org.
HDDS-559. fs.default.name is deprecated.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/911b570d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/911b570d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/911b570d
Branch: refs/heads/ozone-0.3
Commit: 911b570d6710e6304ad147e851f2f9ef407f7ee6
Parents: 78ec30c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:57:39 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
hadoop-ozone/docs/content/OzoneFS.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/911b570d/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index e19c9b9..06102a2 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -46,7 +46,7 @@ Please add the following entry to the core-site.xml.
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
</property>
<property>
- <name>fs.default.name</name>
+ <name>fs.defaultFS</name>
<value>o3fs://localhost:9864/volume/bucket</value>
</property>
{{< /highlight >}}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/6] hadoop git commit: HDDS-604. Correct Ozone getOzoneConf
description. Contributed by Dinesh Chitlangia.
Posted by el...@apache.org.
HDDS-604. Correct Ozone getOzoneConf description.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bec17eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bec17eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bec17eb
Branch: refs/heads/ozone-0.3
Commit: 5bec17ebcfb2bb654479e3c458b1cb6590bbb3e7
Parents: e0d0ef5
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:52:06 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100
----------------------------------------------------------------------
hadoop-ozone/common/src/main/bin/ozone | 4 ++--
hadoop-ozone/common/src/main/bin/start-ozone.sh | 10 +++++-----
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 10 +++++-----
hadoop-ozone/docs/content/CommandShell.md | 2 +-
4 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 4b50771..2ba9ea7 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -39,7 +39,7 @@ function hadoop_usage
hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'"
hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
- hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
+ hadoop_add_subcommand "getconf" client "get ozone config values from configuration"
hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "om" daemon "Ozone Manager"
@@ -94,7 +94,7 @@ function ozonecmd_case
HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
- getozoneconf)
+ getconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index cfb54e0..4c022fb 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -67,8 +67,8 @@ fi
#Add other possible options
nameStartOpt="$nameStartOpt $*"
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -77,7 +77,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -96,7 +96,7 @@ hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone ozonemanager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Starting Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -113,7 +113,7 @@ HADOOP_JUMBO_RETCOUNTER=$?
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Starting storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index 97e1df4..fe902eb 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,8 +47,8 @@ else
exit 1
fi
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
echo "Ozone is not supported in a security enabled cluster."
@@ -57,7 +57,7 @@ fi
#---------------------------------------------------------
# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
if [[ "${OZONE_ENABLED}" != "true" ]]; then
echo "Operation is not supported because ozone is not enabled."
exit -1
@@ -76,7 +76,7 @@ hadoop_uservar_su ozone datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone Manager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
echo "Stopping Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
@@ -91,7 +91,7 @@ hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
#---------------------------------------------------------
# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
echo "Stopping storage container manager nodes [${SCM_NODES}]"
hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/docs/content/CommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/CommandShell.md b/hadoop-ozone/docs/content/CommandShell.md
index 74072a5..fdd6c01 100644
--- a/hadoop-ozone/docs/content/CommandShell.md
+++ b/hadoop-ozone/docs/content/CommandShell.md
@@ -36,7 +36,7 @@ The commands supported by ozone are:
* **envvars** - Display computed Hadoop environment variables.
* **freon** - Runs the ozone load generator.
* **genesis** - Developer Only, Ozone micro-benchmark application.
- * **getozoneconf** - Reads ozone config values from configuration.
+ * **getconf** - Reads ozone config values from configuration.
* **jmxget** - Get JMX exported values from NameNode or DataNode.
* **om** - Ozone Manager, via daemon command can be started or stopped.
* **sh** - Primary command line interface for ozone.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org