You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by jo...@apache.org on 2018/10/30 08:48:12 UTC
[incubator-nemo] branch master updated: [NEMO-246] Add javadoc
generation to CI (#136)
This is an automated email from the ASF dual-hosted git repository.
johnyangk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 728a79d [NEMO-246] Add javadoc generation to CI (#136)
728a79d is described below
commit 728a79d6c8af58ea969004f82ee1ead457ba68ce
Author: Jangho Seo <ja...@jangho.io>
AuthorDate: Tue Oct 30 17:48:07 2018 +0900
[NEMO-246] Add javadoc generation to CI (#136)
JIRA: [NEMO-246: Add javadoc generation to CI](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-246)
**Major changes:**
- For nemo-frontend-spark, configured doclint to ignore missing symbols because some symbols are written in Scala.
- Added mvn clean javadoc:javadoc to .travis.yml
**Minor changes to note:**
- Fixed some javadoc errors according to the reports from doclint.
- Added docs/ to .gitignore
- Added docs/ to RAT ignore list
**Tests for the changes:**
- N/A
**Other comments:**
- N/A
Closes #136
---
.gitignore | 1 +
.travis.yml | 1 +
compiler/frontend/spark/pom.xml | 19 +++++++++++++++++++
.../nemo/examples/beam/WriteOneFilePerWindow.java | 2 +-
pom.xml | 2 ++
.../executor/bytetransfer/ClosableBlockingQueue.java | 1 +
.../runtime/executor/data/BroadcastManagerWorker.java | 2 +-
.../nemo/runtime/executor/data/SerializerManager.java | 2 +-
.../data/partitioner/DataSkewHashPartitioner.java | 3 ---
.../executor/data/stores/GlusterFileStore.java | 3 ---
.../runtime/executor/data/stores/LocalFileStore.java | 3 ---
.../nemo/runtime/executor/task/TaskExecutor.java | 3 ++-
.../org/apache/nemo/runtime/master/MetricStore.java | 2 ++
.../org/apache/nemo/runtime/master/RuntimeMaster.java | 1 +
.../runtime/master/resource/ExecutorRepresenter.java | 6 ++++--
.../master/resource/ResourceSpecification.java | 2 +-
16 files changed, 37 insertions(+), 16 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5def0cd..641eb9b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ REEF_YARN_RUNTIME
target
generated
build
+docs/
#
# ----------------------------------------------------------------------
# Files generated by OutputService during runtime
diff --git a/.travis.yml b/.travis.yml
index b98a183..413b2ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,6 +28,7 @@ script:
# the following command line builds the project, runs the tests with coverage and then execute the SonarCloud analysis
- if [ "$TRAVIS_PULL_REQUEST" == false ]; then travis_retry mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent verify sonar:sonar -B -q -ff -Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info; fi
- if [ "$TRAVIS_PULL_REQUEST" != false ]; then travis_retry mvn clean verify -B -q -ff -Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info; fi
+ - mvn clean install -DskipTests && mvn javadoc:javadoc
notifications:
slack:
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index 93f9f4e..c79a8e1 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -115,6 +115,25 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <outputDirectory>docs/apidocs</outputDirectory>
+ <reportOutputDirectory>docs/apidocs</reportOutputDirectory>
+ <doclint>-missing</doclint>
+ </configuration>
+ <executions>
+ <execution>
+ <id>aggregate</id>
+ <goals>
+ <goal>aggregate</goal>
+ </goals>
+ <phase>site</phase>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java b/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
index fe5d5f7..07a652e 100644
--- a/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
+++ b/examples/beam/src/main/java/org/apache/nemo/examples/beam/WriteOneFilePerWindow.java
@@ -60,7 +60,7 @@ public final class WriteOneFilePerWindow extends PTransform<PCollection<String>,
return input.apply(write);
}
/**
- * A {@link FilenamePolicy} produces a base file name for a write based on metadata about the data
+ * A {@link FileBasedSink.FilenamePolicy} produces a base file name for a write based on metadata about the data
* being written. This always includes the shard number and the total number of shards. For
* windowed writes, it also includes the window and pane index (a sequence number assigned to each
* trigger firing).
diff --git a/pom.xml b/pom.xml
index 2db16fe..9d7e769 100644
--- a/pom.xml
+++ b/pom.xml
@@ -284,6 +284,8 @@ under the License.
<exclude>.editorconfig</exclude>
<!-- formatter.xml -->
<exclude>formatter.xml</exclude>
+ <!-- javadoc -->
+ <exclude>**/docs/**</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
index c6a9169..3d992e7 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/bytetransfer/ClosableBlockingQueue.java
@@ -80,6 +80,7 @@ public final class ClosableBlockingQueue<T> implements AutoCloseable {
/**
* Mark the input end of this queue as closed.
+ * @param throwableToSet a throwable to set as the cause
*/
public synchronized void closeExceptionally(final Throwable throwableToSet) {
this.throwable = throwableToSet;
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
index c8a87fc..17a62c5 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/BroadcastManagerWorker.java
@@ -115,7 +115,7 @@ public final class BroadcastManagerWorker {
* (i.e., the variable is expressed as an IREdge, and reside in a executor as a block)
*
* @param id of the broadcast variable.
- * @param inputReader
+ * @param inputReader the {@link InputReader} to register.
*/
public void registerInputReader(final Serializable id,
final InputReader inputReader) {
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
index 6a3c647..dc46ccb 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/SerializerManager.java
@@ -48,7 +48,7 @@ public final class SerializerManager {
/**
* Register a encoderFactory for runtime edge.
- * This method regards that compression & decompression property are empty.
+ * This method regards that compression and decompression property are empty.
*
* @param runtimeEdgeId id of the runtime edge.
* @param encoderFactory the corresponding encoder factory.
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
index cc3eddf..a79defc 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/partitioner/DataSkewHashPartitioner.java
@@ -58,9 +58,6 @@ public final class DataSkewHashPartitioner implements Partitioner<Integer> {
LOG.info("hashRangeBase {} resulting hashRange {}", hashRangeBase, hashRange);
}
- /**
- * @see Partitioner#partition(Object).
- */
@Override
public Integer partition(final Object element) {
return Math.abs(keyExtractor.extractKey(element).hashCode() % hashRange);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
index 8fc1bbd..a47afea 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/GlusterFileStore.java
@@ -62,9 +62,6 @@ public final class GlusterFileStore extends AbstractBlockStore implements Remote
new File(fileDirectory).mkdirs();
}
- /**
- * @see BlockStore#createBlock(String).
- */
@Override
public Block createBlock(final String blockId) {
deleteBlock(blockId);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
index 7d9f2ed..62758aa 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/stores/LocalFileStore.java
@@ -53,9 +53,6 @@ public final class LocalFileStore extends LocalBlockStore {
new File(fileDirectory).mkdirs();
}
- /**
- * @see BlockStore#createBlock(String).
- */
@Override
public Block createBlock(final String blockId) {
deleteBlock(blockId);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
index 9600410..a2da4bc 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/task/TaskExecutor.java
@@ -92,7 +92,8 @@ public final class TaskExecutor {
* @param taskStateManager State manager for this Task.
* @param intermediateDataIOFactory For reading from/writing to data to other tasks.
* @param broadcastManagerWorker For broadcasts.
- * @param metricMessageSender For sending metric with execution stats to Master.
+ * @param metricMessageSender For sending metric with execution stats to the master.
+ * @param persistentConnectionToMasterMap For sending messages to the master.
*/
public TaskExecutor(final Task task,
final DAG<IRVertex, RuntimeEdge<IRVertex>> irVertexDag,
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java b/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
index 0fd5492..42f22aa 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/MetricStore.java
@@ -150,6 +150,7 @@ public final class MetricStore {
/**
* Dumps JSON-serialized string of specific metric.
* @param metricClass class of metric.
+ * @param <T> type of the metric to dump
* @return dumped JSON string of all metric.
* @throws IOException when failed to write json.
*/
@@ -223,6 +224,7 @@ public final class MetricStore {
* send changed metric data to the frontend client. Also this method is synchronized.
* @param metricClass class of the metric.
* @param id id of the metric.
+ * @param <T> type of the metric to broadcast
*/
public synchronized <T extends Metric> void triggerBroadcast(final Class<T> metricClass, final String id) {
final MetricBroadcaster metricBroadcaster = MetricBroadcaster.getInstance();
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
index 3f1dd3c..d20e854 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/RuntimeMaster.java
@@ -164,6 +164,7 @@ public final class RuntimeMaster {
*
* @param plan to execute
* @param maxScheduleAttempt the max number of times this plan/sub-part of the plan should be attempted.
+ * @return pair of {@link PlanStateManager} and {@link ScheduledExecutorService}
*/
public Pair<PlanStateManager, ScheduledExecutorService> execute(final PhysicalPlan plan,
final int maxScheduleAttempt) {
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
index 351542a..63d98d5 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ExecutorRepresenter.java
@@ -91,6 +91,8 @@ public final class ExecutorRepresenter {
/**
* Marks all Tasks which were running in this executor as failed.
+ *
+ * @return set of identifiers of tasks that were running in this executor.
*/
public Set<String> onExecutorFailed() {
failedTasks.addAll(runningComplyingTasks.values());
@@ -104,7 +106,7 @@ public final class ExecutorRepresenter {
/**
* Marks the Task as running, and sends scheduling message to the executor.
- * @param task
+ * @param task the task to run
*/
public void onTaskScheduled(final Task task) {
(task.getPropertyValue(ResourceSlotProperty.class).orElse(true)
@@ -136,7 +138,7 @@ public final class ExecutorRepresenter {
/**
* Marks the specified Task as completed.
- *
+ * @param taskId id of the completed task
*/
public void onTaskExecutionComplete(final String taskId) {
final Task completedTask = removeFromRunningTasks(taskId);
diff --git a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
index e320338..58fe251 100644
--- a/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
+++ b/runtime/master/src/main/java/org/apache/nemo/runtime/master/resource/ResourceSpecification.java
@@ -75,7 +75,7 @@ public final class ResourceSpecification {
/**
* @return -1 if this resource is not poisoned. (for all other normal cases)
- * >= 0 the expected time to failure by poison. (for fault-handling tests)
+ * >= 0 the expected time to failure by poison. (for fault-handling tests)
*/
public int getPoisonSec() {
return poisonSec;