You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2019/04/26 10:33:27 UTC
[hadoop] branch HDDS-1469 created (now 28dc12b)
This is an automated email from the ASF dual-hosted git repository.
elek pushed a change to branch HDDS-1469
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
at 28dc12b fix unit tests
This branch includes the following new commits:
new 4c7d7f4 address review comments
new 9038996 RAT and javadoc fixes
new 64ff28f reestore ozone.scm.heartbeat.thread.interval
new 28dc12b fix unit tests
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/04: address review comments
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1469
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4c7d7f47b76a3244448f6a52ae469d25f5a6f13c
Author: Márton Elek <el...@apache.org>
AuthorDate: Fri Apr 26 12:03:50 2019 +0200
address review comments
---
.../hadoop/hdds/conf/SimpleConfiguration.java | 15 ++++++++-----
.../java/org/apache/hadoop/hdds/conf/Config.java | 4 ++--
.../hadoop/hdds/conf/ConfigFileAppender.java | 4 ++--
.../hadoop/hdds/conf/ConfigFileGenerator.java | 3 +--
.../hadoop/hdds/conf/ConfigurationExample.java | 26 +++++++++++++---------
5 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java
index cb87247..f18fd5e 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/SimpleConfiguration.java
@@ -35,28 +35,33 @@ public class SimpleConfiguration {
private long waitTime = 1;
- @Config(key = "address", defaultValue = "localhost")
+ @Config(key = "address", defaultValue = "localhost", description = "Just "
+ + "for testing", tags = ConfigTag.MANAGEMENT)
public void setClientAddress(String clientAddress) {
this.clientAddress = clientAddress;
}
- @Config(key = "bind.host", defaultValue = "0.0.0.0")
+ @Config(key = "bind.host", defaultValue = "0.0.0.0", description = "Just "
+ + "for testing", tags = ConfigTag.MANAGEMENT)
public void setBindHost(String bindHost) {
this.bindHost = bindHost;
}
- @Config(key = "enabled", defaultValue = "true")
+ @Config(key = "enabled", defaultValue = "true", description = "Just for "
+ + "testing", tags = ConfigTag.MANAGEMENT)
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
- @Config(key = "port", defaultValue = "9878")
+ @Config(key = "port", defaultValue = "9878", description = "Just for "
+ + "testing", tags = ConfigTag.MANAGEMENT)
public void setPort(int port) {
this.port = port;
}
@Config(key = "wait", type = ConfigType.TIME, timeUnit =
- TimeUnit.SECONDS, defaultValue = "10m")
+ TimeUnit.SECONDS, defaultValue = "10m", description = "Just for "
+ + "testing", tags = ConfigTag.MANAGEMENT)
public void setWaitTime(long waitTime) {
this.waitTime = waitTime;
}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
index 15b60ad..70aa58d 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/Config.java
@@ -43,7 +43,7 @@ public @interface Config {
/**
* Custom description as a help.
*/
- String description() default "";
+ String description();
/**
* Type of configuration. Use AUTO to decide it based on the java type.
@@ -55,5 +55,5 @@ public @interface Config {
*/
TimeUnit timeUnit() default TimeUnit.MILLISECONDS;
- ConfigTag[] tags() default {ConfigTag.OZONE};
+ ConfigTag[] tags();
}
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java
index d88554a..9463f42 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileAppender.java
@@ -36,8 +36,8 @@ import org.w3c.dom.Element;
/**
* Simple DOM based config file writer.
* <p>
- * This class can init/load existing ozone-site.xml fragments and append
- * new entries and write to the file system.
+ * This class can init/load existing ozone-default-generated.xml fragments
+ * and append new entries and write to the file system.
*/
public class ConfigFileAppender {
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java
index a6a5c9a..76cf1f2 100644
--- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigFileGenerator.java
@@ -37,7 +37,7 @@ import java.util.Set;
/**
* Annotation processor to generate ozone-site-generated fragments from
- * ozone-site.xml.
+ * @Config annotations.
*/
@SupportedAnnotationTypes("org.apache.hadoop.hdds.conf.ConfigGroup")
public class ConfigFileGenerator extends AbstractProcessor {
@@ -52,7 +52,6 @@ public class ConfigFileGenerator extends AbstractProcessor {
}
Filer filer = processingEnv.getFiler();
- System.out.println("round");
try {
diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java
index 8ef8191..2dd2669 100644
--- a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java
+++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/ConfigurationExample.java
@@ -29,34 +29,40 @@ public class ConfigurationExample {
private String bindHost;
- private boolean enabled;
+ private boolean compressionEnabled;
private int port = 1234;
private long waitTime = 1;
- @Config(key = "address", defaultValue = "localhost")
+ @Config(key = "address", defaultValue = "localhost", description = "Client "
+ + "addres (To test string injection).", tags = ConfigTag.MANAGEMENT)
public void setClientAddress(String clientAddress) {
this.clientAddress = clientAddress;
}
- @Config(key = "bind.host", defaultValue = "0.0.0.0")
+ @Config(key = "bind.host", defaultValue = "0.0.0.0", description = "Bind "
+ + "host(To test string injection).", tags = ConfigTag.MANAGEMENT)
public void setBindHost(String bindHost) {
this.bindHost = bindHost;
}
- @Config(key = "enabled", defaultValue = "true")
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
+ @Config(key = "compression.enabled", defaultValue = "true", description =
+ "Compression enabled. (Just to test boolean flag)", tags =
+ ConfigTag.MANAGEMENT)
+ public void setCompressionEnabled(boolean compressionEnabled) {
+ this.compressionEnabled = compressionEnabled;
}
- @Config(key = "port", defaultValue = "1234")
+ @Config(key = "port", defaultValue = "1234", description = "Port number "
+ + "config (To test in injection)", tags = ConfigTag.MANAGEMENT)
public void setPort(int port) {
this.port = port;
}
@Config(key = "wait", type = ConfigType.TIME, timeUnit =
- TimeUnit.SECONDS, defaultValue = "30m")
+ TimeUnit.SECONDS, defaultValue = "30m", description = "Wait time (To "
+ + "test TIME config type)", tags = ConfigTag.MANAGEMENT)
public void setWaitTime(long waitTime) {
this.waitTime = waitTime;
}
@@ -69,8 +75,8 @@ public class ConfigurationExample {
return bindHost;
}
- public boolean isEnabled() {
- return enabled;
+ public boolean isCompressionEnabled() {
+ return compressionEnabled;
}
public int getPort() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 04/04: fix unit tests
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1469
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 28dc12be7f2b052bdc1ff5d9f3b1b770fb95cbd0
Author: Márton Elek <el...@apache.org>
AuthorDate: Fri Apr 26 12:27:35 2019 +0200
fix unit tests
---
.../hadoop/hdds/conf/OzoneConfiguration.java | 4 +++-
.../src/test/resources/core-site.xml | 24 ++++++++++++++++++++++
.../src/test/resources/hdfs-site.xml | 24 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index 99477d4..b32ad63 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -55,7 +55,9 @@ public class OzoneConfiguration extends Configuration {
super(conf);
//load the configuration from the classloader of the original conf.
setClassLoader(conf.getClassLoader());
- loadDefaults();
+ if (!(conf instanceof OzoneConfiguration)) {
+ loadDefaults();
+ }
}
private void loadDefaults() {
diff --git a/hadoop-ozone/integration-test/src/test/resources/core-site.xml b/hadoop-ozone/integration-test/src/test/resources/core-site.xml
new file mode 100644
index 0000000..77dd7ef
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/resources/core-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
diff --git a/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml b/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml
new file mode 100644
index 0000000..77dd7ef
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/resources/hdfs-site.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 03/04: reestore ozone.scm.heartbeat.thread.interval
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1469
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 64ff28fb1dc404f2c8fca1fa9b006767cf22f232
Author: Márton Elek <el...@apache.org>
AuthorDate: Fri Apr 26 12:14:31 2019 +0200
reestore ozone.scm.heartbeat.thread.interval
---
.../common/src/main/resources/ozone-default.xml | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 1dcbdc5..05bd7fe 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -961,7 +961,25 @@
Timeout value for the RPC from Datanode to SCM.
</description>
</property>
+ <property>
+ <name>ozone.scm.heartbeat.thread.interval</name>
+ <value>3s</value>
+ <tag>OZONE, MANAGEMENT</tag>
+ <description>
+ When a heartbeat from the data node arrives on SCM, It is queued for
+ processing with the time stamp of when the heartbeat arrived. There is a
+ heartbeat processing thread inside SCM that runs at a specified interval.
+ This value controls how frequently this thread is run.
+ There are some assumptions build into SCM such as this value should allow
+ the heartbeat processing thread to run at least three times more
+ frequently than heartbeats and at least five times more than stale node
+ detection time. If you specify a wrong value, SCM will gracefully refuse
+ to run. For more info look at the node manager tests in SCM.
+
+ In short, you don't need to change this.
+ </description>
+ </property>
<property>
<name>ozone.scm.http-address</name>
<value>0.0.0.0:9876</value>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/04: RAT and javadoc fixes
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch HDDS-1469
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 903899625dde9ede8e7ad588505a164a7bed689f
Author: Márton Elek <el...@apache.org>
AuthorDate: Fri Apr 26 12:05:43 2019 +0200
RAT and javadoc fixes
---
.../org/apache/hadoop/hdds/conf/package-info.java | 22 ++++++++++++++++++++
.../services/javax.annotation.processing.Processor | 15 ++++++++++++++
.../hadoop/hdds/conf/TestConfigFileAppender.java | 20 ++++++++++++++++++
.../org/apache/hadoop/hdds/conf/package-info.java | 24 ++++++++++++++++++++++
.../hdds/scm/container/ReplicationManager.java | 2 --
5 files changed, 81 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
new file mode 100644
index 0000000..e789040
--- /dev/null
+++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Generic configuration annotations, tools and generators.
+ */
+package org.apache.hadoop.hdds.conf;
diff --git a/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor
index fa9740d..f29efda 100644
--- a/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor
+++ b/hadoop-hdds/config/src/main/resources/META-INF/services/javax.annotation.processing.Processor
@@ -1 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
org.apache.hadoop.hdds.conf.ConfigFileGenerator
diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java
index 6b6c0e2..0edb01a 100644
--- a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java
+++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigFileAppender.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hdds.conf;
import java.io.StringWriter;
@@ -5,6 +22,9 @@ import java.io.StringWriter;
import org.junit.Assert;
import org.junit.Test;
+/**
+ * Test the utility which loads/writes the config file fragments.
+ */
public class TestConfigFileAppender {
@Test
diff --git a/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java
new file mode 100644
index 0000000..e8b310d
--- /dev/null
+++ b/hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * Testing configuration tools.
+ */
+
+/**
+ * Testing configuration tools.
+ */
+package org.apache.hadoop.hdds.conf;
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 4b00562..e247e96 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.GeneratedMessage;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.ConfigType;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.Config;
@@ -58,7 +57,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org