You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2018/10/29 14:17:31 UTC

[1/6] hadoop git commit: HDDS-714. Bump protobuf version to 3.5.1. Contributed by Mukul Kumar Singh.

Repository: hadoop
Updated Branches:
  refs/heads/ozone-0.3 095e8f61b -> 031cdf5d5


HDDS-714. Bump protobuf version to 3.5.1. Contributed by Mukul Kumar Singh.

(cherry picked from commit ace06a93baa09293c254d18c709162771738b092)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/031cdf5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/031cdf5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/031cdf5d

Branch: refs/heads/ozone-0.3
Commit: 031cdf5d5f9745516ef1f2b83b91c5ccd45ae736
Parents: 4c71caa
Author: Mukul Kumar Singh <ms...@apache.org>
Authored: Thu Oct 25 09:34:06 2018 +0530
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/031cdf5d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 22c3b35..ec018a6 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -110,7 +110,7 @@
 
     <!-- Maven protoc compiler -->
     <protobuf-maven-plugin.version>0.5.1</protobuf-maven-plugin.version>
-    <protobuf-compile.version>3.5.0</protobuf-compile.version>
+    <protobuf-compile.version>3.5.1</protobuf-compile.version>
     <grpc.version>1.10.0</grpc.version>
     <os-maven-plugin.version>1.5.0.Final</os-maven-plugin.version>
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[6/6] hadoop git commit: HDDS-568. ozone sh volume info, update, delete operations fail when volume name is not prefixed by /. Contributed by Dinesh Chitlangia.

Posted by el...@apache.org.
HDDS-568. ozone sh volume info, update, delete operations fail when volume name is not prefixed by /.
Contributed by Dinesh Chitlangia.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0d0ef58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0d0ef58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0d0ef58

Branch: refs/heads/ozone-0.3
Commit: e0d0ef58db4398903f4962099e4eb2b1b677e5f0
Parents: 911b570
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:16:52 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 .../hadoop/ozone/ozShell/TestOzoneShell.java    | 48 ++++++++++++++++++++
 .../hadoop/ozone/web/ozShell/Handler.java       | 28 ++++++++++++
 .../web/ozShell/volume/DeleteVolumeHandler.java | 12 +----
 .../web/ozShell/volume/InfoVolumeHandler.java   | 22 +--------
 .../web/ozShell/volume/UpdateVolumeHandler.java | 12 +----
 5 files changed, 79 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
index c736f81..a7eecc0 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
@@ -305,6 +305,33 @@ public class TestOzoneShell {
       GenericTestUtils.assertExceptionContains(
           "Info Volume failed, error:VOLUME_NOT_FOUND", e);
     }
+
+
+    volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+    volumeArgs = VolumeArgs.newBuilder()
+        .setOwner("bilbo")
+        .setQuota("100TB")
+        .build();
+    client.createVolume(volumeName, volumeArgs);
+    volume = client.getVolumeDetails(volumeName);
+    assertNotNull(volume);
+
+    //volumeName prefixed with /
+    String volumeNameWithSlashPrefix = "/" + volumeName;
+    args = new String[] {"volume", "delete",
+        url + "/" + volumeNameWithSlashPrefix};
+    execute(shell, args);
+    output = out.toString();
+    assertTrue(output.contains("Volume " + volumeName + " is deleted"));
+
+    // verify if volume has been deleted
+    try {
+      client.getVolumeDetails(volumeName);
+      fail("Get volume call should have thrown.");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains(
+          "Info Volume failed, error:VOLUME_NOT_FOUND", e);
+    }
   }
 
   @Test
@@ -317,6 +344,7 @@ public class TestOzoneShell {
         .build();
     client.createVolume(volumeName, volumeArgs);
 
+    //volumeName supplied as-is
     String[] args = new String[] {"volume", "info", url + "/" + volumeName};
     execute(shell, args);
 
@@ -325,6 +353,17 @@ public class TestOzoneShell {
     assertTrue(output.contains("createdOn")
         && output.contains(OzoneConsts.OZONE_TIME_ZONE));
 
+    //volumeName prefixed with /
+    String volumeNameWithSlashPrefix = "/" + volumeName;
+    args = new String[] {"volume", "info",
+        url + "/" + volumeNameWithSlashPrefix};
+    execute(shell, args);
+
+    output = out.toString();
+    assertTrue(output.contains(volumeName));
+    assertTrue(output.contains("createdOn")
+        && output.contains(OzoneConsts.OZONE_TIME_ZONE));
+
     // test infoVolume with invalid volume name
     args = new String[] {"volume", "info",
         url + "/" + volumeName + "/invalid-name"};
@@ -387,6 +426,15 @@ public class TestOzoneShell {
     vol = client.getVolumeDetails(volumeName);
     assertEquals(newUser, vol.getOwner());
 
+    //volume with / prefix
+    String volumeWithPrefix = "/" + volumeName;
+    String newUser2 = "new-user2";
+    args = new String[] {"volume", "update", url + "/" + volumeWithPrefix,
+        "--user", newUser2};
+    execute(shell, args);
+    vol = client.getVolumeDetails(volumeName);
+    assertEquals(newUser2, vol.getOwner());
+
     // test error conditions
     args = new String[] {"volume", "update", url + "/invalid-volume",
         "--user", newUser};

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
index 5067855..be579b3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.ozone.web.ozShell;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.conf.Configuration;
@@ -153,6 +155,32 @@ public abstract class Handler implements Callable<Void> {
     }
   }
 
+  /**
+   *
+   * @param uri
+   * @return volumeName
+   * @throws Exception
+   * @throws OzoneClientException when uri is null or invalid volume name
+   */
+  protected String parseVolumeName(String uri) throws Exception{
+    URI ozoneURI = verifyURI(uri);
+    Path path = Paths.get(ozoneURI.getPath());
+    int pathNameCount = path.getNameCount();
+    if (pathNameCount != 1) {
+      String errorMessage;
+      if (pathNameCount < 1) {
+        errorMessage = "Volume name is required to perform volume " +
+            "operations like info, update, create and delete. ";
+      } else {
+        errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
+            "volume name";
+      }
+      throw new OzoneClientException(errorMessage);
+    }
+
+    return ozoneURI.getPath().replaceAll("^/+", "");
+  }
+
   public boolean isVerbose() {
     return parent.isVerbose();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
index d1e96fc..d757e20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.ozone.web.ozShell.volume;
 
-import java.net.URI;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.web.ozShell.Handler;
 import org.apache.hadoop.ozone.web.ozShell.Shell;
 
@@ -43,14 +40,7 @@ public class DeleteVolumeHandler extends Handler {
   @Override
   public Void call() throws Exception {
 
-    URI ozoneURI = verifyURI(uri);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneClientException(
-          "Volume name is required to delete a volume");
-    }
-
-    // we need to skip the slash in the URI path
-    String volumeName = ozoneURI.getPath().substring(1);
+    String volumeName = parseVolumeName(uri);
 
     if (isVerbose()) {
       System.out.printf("Volume name : %s%n", volumeName);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
index 16c0342..48ed9f6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
@@ -18,11 +18,6 @@
 
 package org.apache.hadoop.ozone.web.ozShell.volume;
 
-import java.net.URI;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.OzoneClientUtils;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.web.ozShell.Handler;
@@ -48,22 +43,7 @@ public class InfoVolumeHandler extends Handler{
   @Override
   public Void call() throws Exception {
 
-    URI ozoneURI = verifyURI(uri);
-    Path path = Paths.get(ozoneURI.getPath());
-    int pathNameCount = path.getNameCount();
-    if (pathNameCount != 1) {
-      String errorMessage;
-      if (pathNameCount < 1) {
-        errorMessage = "Volume name is required to get info of a volume";
-      } else {
-        errorMessage = "Invalid volume name. Delimiters (/) not allowed in " +
-            "volume name";
-      }
-      throw new OzoneClientException(errorMessage);
-    }
-
-    // we need to skip the slash in the URI path
-    String volumeName = ozoneURI.getPath().substring(1);
+    String volumeName = parseVolumeName(uri);
 
     OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
     System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0d0ef58/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
index 0336fc2..46803d8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
@@ -25,13 +25,10 @@ import picocli.CommandLine.Parameters;
 import org.apache.hadoop.hdds.client.OzoneQuota;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.web.ozShell.Handler;
 import org.apache.hadoop.ozone.web.ozShell.Shell;
 import org.apache.hadoop.ozone.web.utils.JsonUtils;
 
-import java.net.URI;
-
 /**
  * Executes update volume calls.
  */
@@ -57,14 +54,7 @@ public class UpdateVolumeHandler extends Handler {
   @Override
   public Void call() throws Exception {
 
-    URI ozoneURI = verifyURI(uri);
-    if (ozoneURI.getPath().isEmpty()) {
-      throw new OzoneClientException(
-          "Volume name is required to update a volume");
-    }
-
-    // we need to skip the slash in the URI path
-    String volumeName = ozoneURI.getPath().substring(1);
+    String volumeName = parseVolumeName(uri);
 
     OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
     if (quota != null && !quota.isEmpty()) {


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[5/6] hadoop git commit: HDDS-443. Create reusable ProgressBar utility for freon tests. Contributed by Zsolt Horvath.

Posted by el...@apache.org.
HDDS-443. Create reusable ProgressBar utility for freon tests.
Contributed by Zsolt Horvath.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c71caa3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c71caa3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c71caa3

Branch: refs/heads/ozone-0.3
Commit: 4c71caa30b8eabb2aedad8b619eb16711547ab91
Parents: 5bec17e
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 18:18:19 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 hadoop-ozone/tools/pom.xml                      |   6 +
 .../apache/hadoop/ozone/freon/ProgressBar.java  | 210 +++++++++++++++++++
 .../hadoop/ozone/freon/TestProgressBar.java     | 112 ++++++++++
 3 files changed, 328 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index b04bbf1..560e072 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -77,6 +77,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <scope>test</scope>
       <type>test-jar</type>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>2.15.0</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
new file mode 100644
index 0000000..a8d7e73
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import java.io.PrintStream;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+/**
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
+ */
+public class ProgressBar {
+
+  private static final long REFRESH_INTERVAL = 1000L;
+
+  private PrintStream stream;
+  private AtomicLong currentValue;
+  private long maxValue;
+  private Thread progressBar;
+  private volatile boolean exception = false;
+  private long startTime;
+
+  /**
+   * @param stream Used to display the progress
+   * @param maxValue Maximum value of the progress
+   */
+  ProgressBar(PrintStream stream, long maxValue) {
+    this.stream = stream;
+    this.maxValue = maxValue;
+    this.currentValue = new AtomicLong(0);
+    this.progressBar = new Thread(new ProgressBarThread());
+  }
+
+  /**
+   * Start a task with a progessbar without any in/out parameters Runnable used
+   * just a task wrapper.
+   *
+   * @param task Runnable
+   */
+  public void start(Runnable task) {
+
+    startTime = System.nanoTime();
+
+    try {
+
+      progressBar.start();
+      task.run();
+
+    } catch (Exception e) {
+      exception = true;
+    } finally {
+
+      try {
+        progressBar.join();
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+    }
+  }
+
+  /**
+   * Start a task with only out parameters.
+   *
+   * @param task Supplier that represents the task
+   * @param <T> Generic return type
+   * @return Whatever the supllier produces
+   */
+  public <T> T start(Supplier<T> task) {
+
+    startTime = System.nanoTime();
+    T result = null;
+
+    try {
+
+      progressBar.start();
+      result = task.get();
+
+    } catch (Exception e) {
+      exception = true;
+    } finally {
+
+      try {
+        progressBar.join();
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+
+      return result;
+    }
+  }
+
+  /**
+   * Start a task with in/out parameters.
+   *
+   * @param input Input of the function
+   * @param task A Function that does the task
+   * @param <T> type of the input
+   * @param <R> return type
+   * @return Whatever the Function returns
+   */
+  public <T, R> R start(T input, Function<T, R> task) {
+
+    startTime = System.nanoTime();
+    R result = null;
+
+    try {
+
+      progressBar.start();
+      result = task.apply(input);
+
+    } catch (Exception e) {
+      exception = true;
+      throw e;
+    } finally {
+
+      try {
+        progressBar.join();
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+
+      return result;
+    }
+  }
+
+  /**
+   * Increment the progress with one step.
+   */
+  public void incrementProgress() {
+    currentValue.incrementAndGet();
+  }
+
+  private class ProgressBarThread implements Runnable {
+
+    @Override
+    public void run() {
+      try {
+
+        stream.println();
+        long value;
+
+        while ((value = currentValue.get()) < maxValue) {
+          print(value);
+
+          if (exception) {
+            break;
+          }
+          Thread.sleep(REFRESH_INTERVAL);
+        }
+
+        if (exception) {
+          stream.println();
+          stream.println("Incomplete termination, " + "check log for " +
+              "exception.");
+        } else {
+          print(maxValue);
+        }
+        stream.println();
+      } catch (InterruptedException e) {
+        stream.println(e);
+      }
+    }
+
+    /**
+     * Given current value prints the progress bar.
+     *
+     * @param value current progress position
+     */
+    private void print(long value) {
+      stream.print('\r');
+      double percent = 100.0 * value / maxValue;
+      StringBuilder sb = new StringBuilder();
+      sb.append(" " + String.format("%.2f", percent) + "% |");
+
+      for (int i = 0; i <= percent; i++) {
+        sb.append('█');
+      }
+      for (int j = 0; j < 100 - percent; j++) {
+        sb.append(' ');
+      }
+      sb.append("|  ");
+      sb.append(value + "/" + maxValue);
+      long timeInSec = TimeUnit.SECONDS.convert(
+          System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+      String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+          (timeInSec % 3600) / 60, timeInSec % 60);
+      sb.append(" Time: " + timeToPrint);
+      stream.print(sb.toString());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c71caa3/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
new file mode 100644
index 0000000..cea7be8
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.PrintStream;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static org.mockito.Mockito.*;
+
+@RunWith(MockitoJUnitRunner.class)
+/**
+ * Tests for the Progressbar class for Freon.
+ */
+public class TestProgressBar {
+
+  private PrintStream stream;
+
+  @Before
+  public void setupMock() {
+    stream = mock(PrintStream.class);
+  }
+
+  @Test
+  public void testWithRunnable() {
+
+    int maxValue = 10;
+
+    ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+    Runnable task = () -> {
+      IntStream.range(0, maxValue).forEach(
+          counter -> {
+            progressbar.incrementProgress();
+          }
+      );
+    };
+
+    progressbar.start(task);
+
+    verify(stream, atLeastOnce()).print(anyChar());
+    verify(stream, atLeastOnce()).print(anyString());
+  }
+
+  @Test
+  public void testWithSupplier() {
+
+    int maxValue = 10;
+
+    ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+    Supplier<Long> tasks = () -> {
+      IntStream.range(0, maxValue).forEach(
+          counter -> {
+            progressbar.incrementProgress();
+          }
+      );
+
+      return 1L; //return the result of the dummy task
+    };
+
+    progressbar.start(tasks);
+
+    verify(stream, atLeastOnce()).print(anyChar());
+    verify(stream, atLeastOnce()).print(anyString());
+  }
+
+  @Test
+  public void testWithFunction() {
+
+    int maxValue = 10;
+    Long result;
+
+    ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+    Function<Long, String> task = (Long l) -> {
+      IntStream.range(0, maxValue).forEach(
+          counter -> {
+            progressbar.incrementProgress();
+          }
+      );
+
+      return "dummy result"; //return the result of the dummy task
+    };
+
+    progressbar.start(1L, task);
+
+    verify(stream, atLeastOnce()).print(anyChar());
+    verify(stream, atLeastOnce()).print(anyString());
+  }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[3/6] hadoop git commit: HDDS-478. Log files related to each daemon doesn't have proper startup and shutdown logs. Contributed by Dinesh Chitlangia.

Posted by el...@apache.org.
HDDS-478. Log files related to each daemon doesn't have proper startup and shutdown logs.
Contributed by Dinesh Chitlangia.

(cherry picked from commit c1fe657a106aaae3bdf81fa4add70962aaee165b)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78ec30cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78ec30cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78ec30cf

Branch: refs/heads/ozone-0.3
Commit: 78ec30cf4e6ce7078ecf8144dece0c34c40150ea
Parents: 095e8f6
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:44:32 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 .../common/src/main/conf/om-audit-log4j2.properties   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ec30cf/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
index 7d097a0..7be51ac 100644
--- a/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
+++ b/hadoop-ozone/common/src/main/conf/om-audit-log4j2.properties
@@ -52,11 +52,15 @@ filter.write.onMismatch=NEUTRAL
 # TRACE (least specific, a lot of data)
 # ALL (least specific, all data)
 
-appenders=console, rolling
-appender.console.type=Console
-appender.console.name=STDOUT
-appender.console.layout.type=PatternLayout
-appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+# Uncomment following section to enable logging to console appender also
+#appenders=console, rolling
+#appender.console.type=Console
+#appender.console.name=STDOUT
+#appender.console.layout.type=PatternLayout
+#appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+
+# Comment this line when using both console and rolling appenders
+appenders=rolling
 
 #Rolling File Appender with size & time thresholds.
 #Rolling is triggered when either threshold is breached.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[4/6] hadoop git commit: HDDS-559. fs.default.name is deprecated. Contributed by Dinesh Chitlangia.

Posted by el...@apache.org.
HDDS-559. fs.default.name is deprecated.
Contributed by  Dinesh Chitlangia.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/911b570d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/911b570d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/911b570d

Branch: refs/heads/ozone-0.3
Commit: 911b570d6710e6304ad147e851f2f9ef407f7ee6
Parents: 78ec30c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 16:57:39 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 hadoop-ozone/docs/content/OzoneFS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/911b570d/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index e19c9b9..06102a2 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -46,7 +46,7 @@ Please add the following entry to the core-site.xml.
   <value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
 </property>
 <property>
-  <name>fs.default.name</name>
+  <name>fs.defaultFS</name>
   <value>o3fs://localhost:9864/volume/bucket</value>
 </property>
 {{< /highlight >}}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


[2/6] hadoop git commit: HDDS-604. Correct Ozone getOzoneConf description. Contributed by Dinesh Chitlangia.

Posted by el...@apache.org.
HDDS-604. Correct Ozone getOzoneConf description.
Contributed by Dinesh Chitlangia.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bec17eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bec17eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bec17eb

Branch: refs/heads/ozone-0.3
Commit: 5bec17ebcfb2bb654479e3c458b1cb6590bbb3e7
Parents: e0d0ef5
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Oct 9 17:52:06 2018 -0700
Committer: Márton Elek <el...@apache.org>
Committed: Mon Oct 29 15:07:36 2018 +0100

----------------------------------------------------------------------
 hadoop-ozone/common/src/main/bin/ozone          |  4 ++--
 hadoop-ozone/common/src/main/bin/start-ozone.sh | 10 +++++-----
 hadoop-ozone/common/src/main/bin/stop-ozone.sh  | 10 +++++-----
 hadoop-ozone/docs/content/CommandShell.md       |  2 +-
 4 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 4b50771..2ba9ea7 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -39,7 +39,7 @@ function hadoop_usage
   hadoop_add_subcommand "fs" client "run a filesystem command on Ozone file system. Equivalent to 'hadoop fs'"
   hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
   hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
-  hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
+  hadoop_add_subcommand "getconf" client "get ozone config values from configuration"
   hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
   hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
   hadoop_add_subcommand "om" daemon "Ozone Manager"
@@ -94,7 +94,7 @@ function ozonecmd_case
       HADOOP_CLASSNAME=org.apache.hadoop.ozone.genesis.Genesis
       OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
     ;;
-    getozoneconf)
+    getconf)
       HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
       OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools"
     ;;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index cfb54e0..4c022fb 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -67,8 +67,8 @@ fi
 #Add other possible options
 nameStartOpt="$nameStartOpt $*"
 
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
 
 if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
   echo "Ozone is not supported in a security enabled cluster."
@@ -77,7 +77,7 @@ fi
 
 #---------------------------------------------------------
 # Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
 if [[ "${OZONE_ENABLED}" != "true" ]]; then
   echo "Operation is not supported because ozone is not enabled."
   exit -1
@@ -96,7 +96,7 @@ hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
 
 #---------------------------------------------------------
 # Ozone ozonemanager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
 echo "Starting Ozone Manager nodes [${OM_NODES}]"
 if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
   OM_NODES=$(hostname)
@@ -113,7 +113,7 @@ HADOOP_JUMBO_RETCOUNTER=$?
 
 #---------------------------------------------------------
 # Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
 echo "Starting storage container manager nodes [${SCM_NODES}]"
 hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
   --workers \

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index 97e1df4..fe902eb 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,8 +47,8 @@ else
   exit 1
 fi
 
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
 
 if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
   echo "Ozone is not supported in a security enabled cluster."
@@ -57,7 +57,7 @@ fi
 
 #---------------------------------------------------------
 # Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
+OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
 if [[ "${OZONE_ENABLED}" != "true" ]]; then
   echo "Operation is not supported because ozone is not enabled."
   exit -1
@@ -76,7 +76,7 @@ hadoop_uservar_su ozone datanode "${HADOOP_HDFS_HOME}/bin/ozone" \
 
 #---------------------------------------------------------
 # Ozone Manager nodes
-OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
+OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -ozonemanagers 2>/dev/null)
 echo "Stopping Ozone Manager nodes [${OM_NODES}]"
 if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
   OM_NODES=$(hostname)
@@ -91,7 +91,7 @@ hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
 
 #---------------------------------------------------------
 # Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -storagecontainermanagers 2>/dev/null)
+SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -storagecontainermanagers 2>/dev/null)
 echo "Stopping storage container manager nodes [${SCM_NODES}]"
 hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/ozone" \
   --workers \

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bec17eb/hadoop-ozone/docs/content/CommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/CommandShell.md b/hadoop-ozone/docs/content/CommandShell.md
index 74072a5..fdd6c01 100644
--- a/hadoop-ozone/docs/content/CommandShell.md
+++ b/hadoop-ozone/docs/content/CommandShell.md
@@ -36,7 +36,7 @@ The commands supported by ozone are:
    * **envvars** - Display computed Hadoop environment variables.
    * **freon** -  Runs the ozone load generator.
    * **genesis**  - Developer Only, Ozone micro-benchmark application.
-   * **getozoneconf** -  Reads ozone config values from configuration.
+   * **getconf** -  Reads ozone config values from configuration.
    * **jmxget**  - Get JMX exported values from NameNode or DataNode.
    * **om** -   Ozone Manager, via daemon command can be started or stopped.
    * **sh** -  Primary command line interface for ozone.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org