You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nemo.apache.org by jo...@apache.org on 2020/02/05 00:37:21 UTC
[incubator-nemo] branch master updated: [NEMO-437] Support Java
version 11 (#281)
This is an automated email from the ASF dual-hosted git repository.
johnyangk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-nemo.git
The following commit(s) were added to refs/heads/master by this push:
new 75e6700 [NEMO-437] Support Java version 11 (#281)
75e6700 is described below
commit 75e6700f5260621a04668aa81902e97fa22ada82
Author: Won Wook SONG <wo...@apache.org>
AuthorDate: Wed Feb 5 09:37:13 2020 +0900
[NEMO-437] Support Java version 11 (#281)
JIRA: [NEMO-437: Support Java version 11](https://issues.apache.org/jira/projects/NEMO/issues/NEMO-437)
**Major changes:**
- Bump java libraries to recent versions
- `beam` 2.11.0 --> 2.18.0
- `grpc` 1.7.0 --> 1.26.0
- `mockito` 2.13.0 --> 3.2.4
- `powermock` 2.0.0-beta --> 2.0.4
- `spark` 2.2.0 --> 2.4.4 (with `scala` 2.11.0 --> 2.12.8)
- as well as version upgrades for `maven-surefire`, `maven-shade`, `maven-deploy`, `maven-javadoc` plugins and `guava`, `jackson`, `sqlite-jdbc`, `PostgreSQL`, `slf4j`, and `junit`.
**Minor changes to note:**
- Syntax changes with the java library updates
**Tests for the changes:**
- I've included new JDK requirements to the travis script
- Existing tests pass
**Other comments:**
- I've referred to the following PRs for upgrading Beam: [link1](https://github.com/apache/beam/pull/7635/files), [link2](https://github.com/apache/beam/pull/9275/files)
Closes #281
---
.travis.yml | 57 +++++++++----------
README.md | 19 ++++---
compiler/frontend/beam/pom.xml | 2 +-
.../compiler/frontend/beam/PipelineTranslator.java | 24 ++++----
.../beam/transform/AbstractDoFnTransform.java | 15 ++++-
.../beam/transform/CreateViewTransform.java | 6 ++
.../frontend/beam/transform/DoFnTransform.java | 10 +++-
.../GroupByKeyAndWindowDoFnTransform.java | 5 +-
.../beam/transform/PushBackDoFnTransform.java | 8 ++-
compiler/frontend/spark/pom.xml | 12 ++--
.../beam/transform/CreateViewTransformTest.java | 9 ++-
.../frontend/beam/transform/DoFnTransformTest.java | 21 +++++--
examples/beam/pom.xml | 6 +-
.../examples/beam/WindowedWordCountITCase.java | 2 -
examples/nexmark/pom.xml | 2 +-
examples/spark/pom.xml | 10 ++--
pom.xml | 51 ++++++++++-------
runtime/common/pom.xml | 5 ++
.../common/message/grpc/GrpcMessageClient.java | 2 +-
.../nemo/runtime/executor/data/MemoryChunk.java | 6 +-
webui/package-lock.json | 64 ++++++++++++++++------
21 files changed, 217 insertions(+), 119 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index d1a500c..bb50300 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,39 +17,34 @@
# under the License.
# .travis.yml
-sudo: required
dist: trusty
-matrix:
- include:
- # For maven builds
- - language: java
- jdk: oraclejdk8
- before_install:
- - sudo apt update && sudo apt install -y libgif-dev
- install: true
- script:
- # the following command line builds the project, runs the tests with coverage and then execute the SonarCloud analysis if it's not an external PR (It's to be supported later on)
- - if [ "$TRAVIS_PULL_REQUEST" == false ]; then travis_retry mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent $GOAL sonar:sonar -Dsonar.host.url=https://sonarcloud.io $SONAR_PROJECT_KEY; fi
- - if [ "$TRAVIS_PULL_REQUEST" != false ]; then travis_retry mvn clean $GOAL; fi
- cache:
- directories:
- - '$HOME/.m2/repository'
- - '$HOME/.sonar/cache'
- env:
- - HADOOP_HOME="/"
- - GOAL="validate"
- - GOAL="verify -B -q -ff -Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info"
-
- # For web ui build
- - language: node_js
- node_js:
- - node
- before_install:
- - pushd webui
- - npm install
- script:
- - npm test && popd
+# For maven builds
+language: java
+jdk:
+ - oraclejdk8
+ - oraclejdk11
+ - openjdk11
+env:
+ global:
+ - HADOOP_HOME="/"
+ jobs:
+ - GOAL="verify -B -q -ff -Dsurefire.useFile=false -Dorg.slf4j.simpleLogger.defaultLogLevel=info"
+before_install:
+ - sudo apt update && sudo apt install -y libgif-dev
+ - nvm install node # for Web UI build
+install: true
+script:
+ # the following command line builds the project, runs the tests with coverage and then execute the SonarCloud analysis if it's not an external PR (It's to be supported later on)
+ - if [ "$TRAVIS_PULL_REQUEST" == false ]; then travis_retry mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent $GOAL sonar:sonar -Dsonar.host.url=https://sonarcloud.io $SONAR_PROJECT_KEY; fi
+ - if [ "$TRAVIS_PULL_REQUEST" != false ]; then travis_retry mvn clean $GOAL; fi
+ # Web UI Build
+ - pushd webui && npm install && npm test && popd
+cache:
+ directories:
+ - '$HOME/.m2/repository'
+ - '$HOME/.sonar/cache'
+ - 'webui/node_modules' # for Web UI build
notifications:
slack:
diff --git a/README.md b/README.md
index 96bc683..85ac631 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ Please refer to the [Contribution guideline](.github/CONTRIBUTING.md) to contrib
## Nemo prerequisites and setup
### Prerequisites
-* Java 8
+* Java 8 or later (tested on Java 8 and Java 11)
* Maven
* YARN settings
* Download Hadoop 2.7.2 at https://archive.apache.org/dist/hadoop/common/hadoop-2.7.2/
@@ -45,8 +45,13 @@ Please refer to the [Contribution guideline](.github/CONTRIBUTING.md) to contrib
* On macOS:
```bash
- $ brew tap homebrew/versions
- $ brew install protobuf@2.5
+ $ wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2
+ $ tar xvf protobuf-2.5.0.tar.bz2
+ $ pushd protobuf-2.5.0
+ $ ./configure CC=clang CXX=clang++ CXXFLAGS='-std=c++11 -stdlib=libc++ -O3 -g' LDFLAGS='-stdlib=libc++' LIBS="-lc++ -lc++abi"
+ $ make -j 4
+ $ sudo make install
+ $ popd
```
* Or build from source:
@@ -66,7 +71,7 @@ Please refer to the [Contribution guideline](.github/CONTRIBUTING.md) to contrib
## Running Beam applications
-Apache Nemo is an official runner of Apache Beam, and it can be executed from Beam, using NemoRunner, as well as directly from the Nemo project.
+Apache Nemo is an official runner of Apache Beam, and it can be executed from Beam, using NemoRunner, as well as directly from the Nemo project.
The details of using NemoRunner from Beam is shown on the [NemoRunner page of the Apache Beam website](https://beam.apache.org/documentation/runners/nemo/).
Below describes how Beam applications can be run directly on Nemo.
@@ -105,13 +110,13 @@ $ ./bin/run_beam.sh \
-optimization_policy org.apache.nemo.compiler.optimizer.policy.TransientResourcePolicy \
-user_args "hdfs://v-m:9000/test_input_wordcount hdfs://v-m:9000/test_output_wordcount"
-## NEXMark streaming Q0 (query0) example
+## NEXMark streaming Q0 (query0) example
$ ./bin/run_nexmark.sh \
-job_id nexmark-Q0 \
-executor_json `pwd`/examples/resources/executors/beam_test_executor_resources.json \
-user_main org.apache.beam.sdk.nexmark.Main \
-optimization_policy org.apache.nemo.compiler.optimizer.policy.StreamingPolicy \
- -scheduler_impl_class_name org.apache.nemo.runtime.master.scheduler.StreamingScheduler \
+ -scheduler_impl_class_name org.apache.nemo.runtime.master.scheduler.StreamingScheduler \
-user_args "--runner=NemoRunner --streaming=true --query=0 --numEventGenerators=1"
```
@@ -171,7 +176,7 @@ $ ./bin/run_beam.sh \
* `-db_id` : ID of the DB from the given address.
* `-db_password`: Credentials for the DB from the given address.
-## Speeding up builds
+## Speeding up builds
* To exclude Spark related packages: mvn clean install -T 2C -DskipTests -pl \\!compiler/frontend/spark,\\!examples/spark
* To exclude Beam related packages: mvn clean install -T 2C -DskipTests -pl \\!compiler/frontend/beam,\\!examples/beam
* To exclude NEXMark related packages: mvn clean install -T 2C -DskipTests -pl \\!examples/nexmark
diff --git a/compiler/frontend/beam/pom.xml b/compiler/frontend/beam/pom.xml
index ee63db7..7a455d3 100644
--- a/compiler/frontend/beam/pom.xml
+++ b/compiler/frontend/beam/pom.xml
@@ -50,7 +50,7 @@ under the License.
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
+ <artifactId>beam-sdks-java-io-hadoop-format</artifactId>
<version>${beam.version}</version>
</dependency>
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
index 2e38b2e..e18f4ae 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/PipelineTranslator.java
@@ -57,9 +57,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -223,7 +221,7 @@ final class PipelineTranslator {
private static void parDoSingleOutputTranslator(final PipelineTranslationContext ctx,
final TransformHierarchy.Node beamNode,
final ParDo.SingleOutput<?, ?> transform) {
- final Map<Integer, PCollectionView<?>> sideInputMap = getSideInputMap(transform.getSideInputs());
+ final Map<Integer, PCollectionView<?>> sideInputMap = getSideInputMap(transform.getSideInputs().values());
final AbstractDoFnTransform doFnTransform = createDoFnTransform(ctx, beamNode, sideInputMap);
final IRVertex vertex = new OperatorVertex(doFnTransform);
@@ -244,7 +242,7 @@ final class PipelineTranslator {
private static void parDoMultiOutputTranslator(final PipelineTranslationContext ctx,
final TransformHierarchy.Node beamNode,
final ParDo.MultiOutput<?, ?> transform) {
- final Map<Integer, PCollectionView<?>> sideInputMap = getSideInputMap(transform.getSideInputs());
+ final Map<Integer, PCollectionView<?>> sideInputMap = getSideInputMap(transform.getSideInputs().values());
final AbstractDoFnTransform doFnTransform = createDoFnTransform(ctx, beamNode, sideInputMap);
final IRVertex vertex = new OperatorVertex(doFnTransform);
ctx.addVertex(vertex);
@@ -420,8 +418,9 @@ final class PipelineTranslator {
* @param viewList list of {@link PCollectionView}s.
* @return map of side inputs.
*/
- private static Map<Integer, PCollectionView<?>> getSideInputMap(final List<PCollectionView<?>> viewList) {
- return IntStream.range(0, viewList.size()).boxed().collect(Collectors.toMap(Function.identity(), viewList::get));
+ private static Map<Integer, PCollectionView<?>> getSideInputMap(final Collection<PCollectionView<?>> viewList) {
+ final PrimitiveIterator.OfInt iterator = IntStream.range(0, viewList.size()).iterator();
+ return viewList.stream().collect(Collectors.toMap(i -> iterator.next(), Function.identity()));
}
/**
@@ -443,6 +442,8 @@ final class PipelineTranslator {
Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform));
final HasDisplayData displayData = (builder) -> builder.add(DisplayData.item("name", beamNode.getFullName()));
+ final DoFnSchemaInformation doFnSchemaInformation =
+ ParDoTranslation.getSchemaInformation(beamNode.toAppliedPTransform(ctx.getPipeline()));
if (sideInputMap.isEmpty()) {
return new DoFnTransform(
@@ -453,7 +454,9 @@ final class PipelineTranslator {
additionalOutputTags.getAll(),
mainInput.getWindowingStrategy(),
ctx.getPipelineOptions(),
- DisplayData.from(displayData));
+ DisplayData.from(displayData),
+ doFnSchemaInformation,
+ Collections.emptyMap());
} else {
return new PushBackDoFnTransform(
doFn,
@@ -464,8 +467,9 @@ final class PipelineTranslator {
mainInput.getWindowingStrategy(),
sideInputMap,
ctx.getPipelineOptions(),
- DisplayData.from(displayData));
-
+ DisplayData.from(displayData),
+ doFnSchemaInformation,
+ Collections.emptyMap());
}
} catch (final IOException e) {
throw new RuntimeException(e);
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
index b3abace..9a50bb5 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/AbstractDoFnTransform.java
@@ -23,6 +23,7 @@ import org.apache.beam.runners.core.construction.SerializablePipelineOptions;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.reflect.DoFnInvoker;
import org.apache.beam.sdk.transforms.reflect.DoFnInvokers;
@@ -82,6 +83,10 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
private boolean bundleFinished = true;
private final DisplayData displayData;
+ private final DoFnSchemaInformation doFnSchemaInformation;
+
+ private final Map<String, PCollectionView<?>> sideInputMapping;
+
/**
* AbstractDoFnTransform constructor.
*
@@ -103,7 +108,9 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
final WindowingStrategy<?, ?> windowingStrategy,
final Map<Integer, PCollectionView<?>> sideInputs,
final PipelineOptions options,
- final DisplayData displayData) {
+ final DisplayData displayData,
+ final DoFnSchemaInformation doFnSchemaInformation,
+ final Map<String, PCollectionView<?>> sideInputMapping) {
this.doFn = doFn;
this.inputCoder = inputCoder;
this.outputCoders = outputCoders;
@@ -113,6 +120,8 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
this.serializedOptions = new SerializablePipelineOptions(options);
this.windowingStrategy = windowingStrategy;
this.displayData = displayData;
+ this.doFnSchemaInformation = doFnSchemaInformation;
+ this.sideInputMapping = sideInputMapping;
}
/**
@@ -285,7 +294,9 @@ public abstract class AbstractDoFnTransform<InputT, InterT, OutputT> implements
stepContext,
inputCoder,
outputCoders,
- windowingStrategy);
+ windowingStrategy,
+ doFnSchemaInformation,
+ sideInputMapping);
pushBackRunner = sideInputs.isEmpty()
? null
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
index bf0e274..f5d59e3 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransform.java
@@ -138,6 +138,12 @@ public final class CreateViewTransform<I, O> implements Transform<WindowedValue<
}
@Override
+ public Iterable<Void> get() {
+ // as it is a iterable of voids, return null.
+ return null;
+ }
+
+ @Override
public Iterable<T> get(@Nullable final Void aVoid) {
return iterable;
}
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
index 2748b02..c6b8e3b 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransform.java
@@ -21,8 +21,10 @@ package org.apache.nemo.compiler.frontend.beam.transform;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.util.WindowedValue;
+import org.apache.beam.sdk.values.PCollectionView;
import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.nemo.common.ir.OutputCollector;
@@ -62,9 +64,11 @@ public final class DoFnTransform<InputT, OutputT> extends AbstractDoFnTransform<
final List<TupleTag<?>> additionalOutputTags,
final WindowingStrategy<?, ?> windowingStrategy,
final PipelineOptions options,
- final DisplayData displayData) {
- super(doFn, inputCoder, outputCoders, mainOutputTag,
- additionalOutputTags, windowingStrategy, Collections.emptyMap(), options, displayData);
+ final DisplayData displayData,
+ final DoFnSchemaInformation doFnSchemaInformation,
+ final Map<String, PCollectionView<?>> sideInputMapping) {
+ super(doFn, inputCoder, outputCoders, mainOutputTag, additionalOutputTags, windowingStrategy,
+ Collections.emptyMap(), options, displayData, doFnSchemaInformation, sideInputMapping);
}
@Override
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
index a7c815f..0818270 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/GroupByKeyAndWindowDoFnTransform.java
@@ -22,6 +22,7 @@ import org.apache.beam.runners.core.*;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
@@ -79,7 +80,9 @@ public final class GroupByKeyAndWindowDoFnTransform<K, InputT>
windowingStrategy,
Collections.emptyMap(), /* GBK does not have additional side inputs */
options,
- displayData);
+ displayData,
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
this.keyToValues = new HashMap<>();
this.reduceFn = reduceFn;
this.prevOutputWatermark = new Watermark(Long.MIN_VALUE);
diff --git a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
index 1ae43b1..8068f66 100644
--- a/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
+++ b/compiler/frontend/beam/src/main/java/org/apache/nemo/compiler/frontend/beam/transform/PushBackDoFnTransform.java
@@ -21,6 +21,7 @@ package org.apache.nemo.compiler.frontend.beam.transform;
import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.util.WindowedValue;
@@ -72,9 +73,12 @@ public final class PushBackDoFnTransform<InputT, OutputT> extends AbstractDoFnTr
final WindowingStrategy<?, ?> windowingStrategy,
final Map<Integer, PCollectionView<?>> sideInputs,
final PipelineOptions options,
- final DisplayData displayData) {
+ final DisplayData displayData,
+ final DoFnSchemaInformation doFnSchemaInformation,
+ final Map<String, PCollectionView<?>> sideInputMapping) {
super(doFn, inputCoder, outputCoders, mainOutputTag,
- additionalOutputTags, windowingStrategy, sideInputs, options, displayData);
+ additionalOutputTags, windowingStrategy, sideInputs, options, displayData,
+ doFnSchemaInformation, sideInputMapping);
this.curPushedBacks = new ArrayList<>();
this.curPushedBackWatermark = Long.MAX_VALUE;
this.curInputWatermark = Long.MIN_VALUE;
diff --git a/compiler/frontend/spark/pom.xml b/compiler/frontend/spark/pom.xml
index a40468a..843600b 100644
--- a/compiler/frontend/spark/pom.xml
+++ b/compiler/frontend/spark/pom.xml
@@ -44,7 +44,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.11</artifactId>
+ <artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
<exclusions>
<exclusion>
@@ -60,7 +60,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.11</artifactId>
+ <artifactId>spark-sql_2.12</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
@@ -75,13 +75,15 @@ under the License.
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
- <version>3.3.1</version>
+ <version>3.3.2</version>
<configuration>
<recompileMode>incremental</recompileMode>
<javacArgs>
<javacArg>-Xlint:unchecked</javacArg>
<javacArg>-Xlint:deprecation</javacArg>
</javacArgs>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
</configuration>
<executions>
<execution>
@@ -104,7 +106,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
+ <version>${maven.compiler.version}</version>
<executions>
<execution>
<phase>compile</phase>
@@ -117,7 +119,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven.javadoc.version}</version>
<configuration>
<outputDirectory>docs/apidocs</outputDirectory>
<reportOutputDirectory>docs/apidocs</reportOutputDirectory>
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
index c04875d..61feac3 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/CreateViewTransformTest.java
@@ -25,6 +25,8 @@ import org.apache.beam.sdk.transforms.windowing.FixedWindows;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
import org.apache.beam.sdk.util.WindowedValue;
import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.nemo.common.ir.vertex.transform.Transform;
import org.apache.nemo.common.punctuation.Watermark;
import org.joda.time.Duration;
@@ -131,7 +133,7 @@ public final class CreateViewTransformTest {
assertEquals(0, oc.outputs.size());
}
- final class SumViewFn extends ViewFn<Materializations.MultimapView<Void, String>, Integer> {
+ final static class SumViewFn extends ViewFn<Materializations.MultimapView<Void, String>, Integer> {
@Override
public Materialization<Materializations.MultimapView<Void, String>> getMaterialization() {
@@ -139,6 +141,11 @@ public final class CreateViewTransformTest {
}
@Override
+ public TypeDescriptor<Integer> getTypeDescriptor() {
+ return TypeDescriptors.integers();
+ }
+
+ @Override
public Integer apply(final Materializations.MultimapView<Void, String> view) {
int sum = 0;
// MultimapView.get is Nullable
diff --git a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
index f343f4d..9321310 100644
--- a/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
+++ b/compiler/test/src/test/java/org/apache/nemo/compiler/frontend/beam/transform/DoFnTransformTest.java
@@ -25,6 +25,7 @@ import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFnSchemaInformation;
import org.apache.beam.sdk.transforms.View;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -77,7 +78,9 @@ public final class DoFnTransformTest {
Collections.emptyList(),
WindowingStrategy.globalDefault(),
PipelineOptionsFactory.as(NemoPipelineOptions.class),
- DisplayData.none());
+ DisplayData.none(),
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
final Transform.Context context = mock(Transform.Context.class);
final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
@@ -111,7 +114,9 @@ public final class DoFnTransformTest {
Collections.emptyList(),
WindowingStrategy.globalDefault(),
pipelineOptions,
- DisplayData.none());
+ DisplayData.none(),
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
final Transform.Context context = mock(Transform.Context.class);
final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
@@ -155,7 +160,9 @@ public final class DoFnTransformTest {
Collections.emptyList(),
WindowingStrategy.globalDefault(),
pipelineOptions,
- DisplayData.none());
+ DisplayData.none(),
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
final Transform.Context context = mock(Transform.Context.class);
final OutputCollector<WindowedValue<String>> oc = new TestOutputCollector<>();
@@ -207,7 +214,9 @@ public final class DoFnTransformTest {
tags,
WindowingStrategy.globalDefault(),
PipelineOptionsFactory.as(NemoPipelineOptions.class),
- DisplayData.none());
+ DisplayData.none(),
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
// mock context
final Transform.Context context = mock(Transform.Context.class);
@@ -267,7 +276,9 @@ public final class DoFnTransformTest {
WindowingStrategy.globalDefault(),
sideInputMap, /* side inputs */
PipelineOptionsFactory.as(NemoPipelineOptions.class),
- DisplayData.none());
+ DisplayData.none(),
+ DoFnSchemaInformation.create(),
+ Collections.emptyMap());
final TestOutputCollector<String> oc = new TestOutputCollector<>();
doFnTransform.prepare(context, oc);
diff --git a/examples/beam/pom.xml b/examples/beam/pom.xml
index 93b3388..8ba7c8e 100644
--- a/examples/beam/pom.xml
+++ b/examples/beam/pom.xml
@@ -51,7 +51,7 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
- <artifactId>beam-sdks-java-io-hadoop-input-format</artifactId>
+ <artifactId>beam-sdks-java-io-hadoop-format</artifactId>
<version>${beam.version}</version>
</dependency>
<dependency>
@@ -87,7 +87,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
- <version>2.8.2</version>
+ <version>${maven.deploy.version}</version>
<configuration>
<skip>true</skip>
</configuration>
@@ -95,7 +95,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven.shade.version}</version>
<executions>
<execution>
<phase>package</phase>
diff --git a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
index 8f37760..3628f7c 100644
--- a/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
+++ b/examples/beam/src/test/java/org/apache/nemo/examples/beam/WindowedWordCountITCase.java
@@ -109,7 +109,6 @@ public final class WindowedWordCountITCase {
}
}
-
@Test(timeout = ExampleTestArgs.TIMEOUT)
public void testStreamingSchedulerAndPipeSlidingWindow() throws Exception {
builder = new ArgBuilder()
@@ -130,7 +129,6 @@ public final class WindowedWordCountITCase {
}
}
-
// TODO #271: We currently disable this test because we cannot force close Nemo
// @Test (timeout = TIMEOUT)
public void testUnboundedSlidingWindow() throws Exception {
diff --git a/examples/nexmark/pom.xml b/examples/nexmark/pom.xml
index 23a4ee1..9fb0a08 100644
--- a/examples/nexmark/pom.xml
+++ b/examples/nexmark/pom.xml
@@ -50,7 +50,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven.shade.version}</version>
<executions>
<execution>
<phase>package</phase>
diff --git a/examples/spark/pom.xml b/examples/spark/pom.xml
index d3e122c..d9fcd57 100644
--- a/examples/spark/pom.xml
+++ b/examples/spark/pom.xml
@@ -71,7 +71,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
- <version>2.8.2</version>
+ <version>${maven.deploy.version}</version>
<configuration>
<skip>true</skip>
</configuration>
@@ -79,7 +79,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven.shade.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -112,13 +112,15 @@ under the License.
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
- <version>3.3.1</version>
+ <version>3.3.2</version>
<configuration>
<recompileMode>incremental</recompileMode>
<javacArgs>
<javacArg>-Xlint:unchecked</javacArg>
<javacArg>-Xlint:deprecation</javacArg>
</javacArgs>
+ <source>1.8</source>
+ <target>1.8</target>
</configuration>
<executions>
<execution>
@@ -141,7 +143,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
+ <version>${maven.compiler.version}</version>
<executions>
<execution>
<phase>compile</phase>
diff --git a/pom.xml b/pom.xml
index 8944e0a..783565b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,35 +41,39 @@ under the License.
</scm>
<properties>
- <maven.compiler.source>1.8</maven.compiler.source>
- <maven.compiler.target>1.8</maven.compiler.target>
+ <java.version>1.8</java.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ <maven.compiler.version>3.8.1</maven.compiler.version>
+ <maven.shade.version>3.2.1</maven.shade.version>
+ <maven.deploy.version>3.0.0-M1</maven.deploy.version>
+ <maven.javadoc.version>3.1.1</maven.javadoc.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<auto-service.version>1.0-rc2</auto-service.version>
- <beam.version>2.11.0</beam.version>
- <spark.version>2.2.0</spark.version>
- <scala.version>2.11.8</scala.version>
- <kryo.version>4.0.1</kryo.version>
+ <beam.version>2.18.0</beam.version>
+ <spark.version>2.4.4</spark.version>
+ <scala.version>2.12.10</scala.version>
+ <kryo.version>4.0.2</kryo.version>
<reef.version>0.16.0</reef.version>
<protobuf.version>2.5.0</protobuf.version>
<hadoop.version>2.7.2</hadoop.version>
<log4j.configuration>file://log4j.properties</log4j.configuration>
<netty.version>4.1.42.Final</netty.version>
- <guava.version>20.0</guava.version>
- <grpc.version>1.7.0</grpc.version>
- <jackson.version>2.8.8</jackson.version>
+ <guava.version>22.0</guava.version>
+ <grpc.version>1.26.0</grpc.version>
+ <jackson.version>2.10.2</jackson.version>
<netlib.version>1.1.2</netlib.version>
- <netty.version>4.1.16.Final</netty.version>
<jetty-server.version>9.4.10.v20180503</jetty-server.version>
<jetty-servlet.version>9.4.10.v20180503</jetty-servlet.version>
<commons-math.version>3.6.1</commons-math.version>
- <sqlite-jdbc.version>3.25.2</sqlite-jdbc.version>
- <postgresql.version>42.2.5</postgresql.version>
- <slf4j.version>1.7.20</slf4j.version>
+ <sqlite-jdbc.version>3.30.1</sqlite-jdbc.version>
+ <postgresql.version>42.2.9</postgresql.version>
+ <slf4j.version>1.7.30</slf4j.version>
<!-- Tests -->
- <mockito.version>2.13.0</mockito.version>
- <powermock.version>2.0.0-beta.5</powermock.version>
- <surefire.version>3.0.0-M1</surefire.version>
- <junit.version>4.12</junit.version>
+ <mockito.version>3.2.4</mockito.version>
+ <powermock.version>2.0.4</powermock.version>
+ <surefire.version>3.0.0-M4</surefire.version>
+ <junit.version>4.13</junit.version>
<checkstyle.version>8.16</checkstyle.version>
</properties>
@@ -151,7 +155,7 @@ under the License.
-->
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
- <version>1.6.2</version>
+ <version>${slf4j.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -169,6 +173,15 @@ under the License.
<pluginManagement>
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven.compiler.version}</version>
+<!-- Uncomment when dropping support for Java 8 -->
+<!-- <configuration>-->
+<!-- <release>${java.version}</release>-->
+<!-- </configuration>-->
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>sonar-maven-plugin</artifactId>
</plugin>
@@ -268,7 +281,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
+ <version>${maven.javadoc.version}</version>
<configuration>
<excludePackageNames>*.org.apache.nemo.runtime.common.comm</excludePackageNames>
<outputDirectory>docs/apidocs</outputDirectory>
diff --git a/runtime/common/pom.xml b/runtime/common/pom.xml
index 00a458f..9912491 100644
--- a/runtime/common/pom.xml
+++ b/runtime/common/pom.xml
@@ -64,6 +64,11 @@ under the License.
<version>${grpc.version}</version>
</dependency>
<dependency>
+ <groupId>javax.annotation</groupId>
+ <artifactId>javax.annotation-api</artifactId>
+ <version>1.3.2</version>
+ </dependency>
+ <dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-testing</artifactId>
<version>${grpc.version}</version>
diff --git a/runtime/common/src/main/java/org/apache/nemo/runtime/common/message/grpc/GrpcMessageClient.java b/runtime/common/src/main/java/org/apache/nemo/runtime/common/message/grpc/GrpcMessageClient.java
index b96e815..71c670d 100644
--- a/runtime/common/src/main/java/org/apache/nemo/runtime/common/message/grpc/GrpcMessageClient.java
+++ b/runtime/common/src/main/java/org/apache/nemo/runtime/common/message/grpc/GrpcMessageClient.java
@@ -90,7 +90,7 @@ final class GrpcMessageClient {
*/
private void setupChannel(final InetSocketAddress ipAddress) throws Exception {
this.managedChannel = ManagedChannelBuilder.forAddress(ipAddress.getHostName(), ipAddress.getPort())
- .usePlaintext(true)
+ .usePlaintext()
.build();
this.blockingStub = MessageServiceGrpc.newBlockingStub(managedChannel);
this.asyncStub = MessageServiceGrpc.newStub(managedChannel);
diff --git a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/MemoryChunk.java b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/MemoryChunk.java
index 8e786ef..ab341da 100644
--- a/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/MemoryChunk.java
+++ b/runtime/executor/src/main/java/org/apache/nemo/runtime/executor/data/MemoryChunk.java
@@ -86,7 +86,7 @@ public class MemoryChunk {
*
* @return {@link ByteBuffer}
*/
- public ByteBuffer getBuffer() {
+ public final ByteBuffer getBuffer() {
if (address > addressLimit) {
throw new IllegalStateException(CHUNK_FREED);
}
@@ -98,14 +98,14 @@ public class MemoryChunk {
*
* @return the MemoryChunk with the same content of the caller instance
*/
- public MemoryChunk duplicate() {
+ public final MemoryChunk duplicate() {
return new MemoryChunk(buffer.duplicate());
}
/**
* Frees this MemoryChunk. No further operation possible after calling this method.
*/
- public void release() {
+ public final void release() {
buffer.clear();
address = addressLimit + 1;
}
diff --git a/webui/package-lock.json b/webui/package-lock.json
index dc85c47..a418e2f 100644
--- a/webui/package-lock.json
+++ b/webui/package-lock.json
@@ -579,7 +579,8 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
- "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
+ "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+ "optional": true
},
"assign-symbols": {
"version": "1.0.0",
@@ -2535,6 +2536,7 @@
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz",
"integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==",
+ "optional": true,
"requires": {
"delayed-stream": "~1.0.0"
}
@@ -3168,7 +3170,8 @@
"cssom": {
"version": "0.3.4",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.4.tgz",
- "integrity": "sha512-+7prCSORpXNeR4/fUP3rL+TzqtiFfhMvTd7uEqMdgPvLPt4+uzFUeufx5RHjGTACCargg/DiEt/moMQmvnfkog=="
+ "integrity": "sha512-+7prCSORpXNeR4/fUP3rL+TzqtiFfhMvTd7uEqMdgPvLPt4+uzFUeufx5RHjGTACCargg/DiEt/moMQmvnfkog==",
+ "optional": true
},
"cssstyle": {
"version": "0.2.37",
@@ -3339,7 +3342,8 @@
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
- "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
+ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
+ "optional": true
},
"depd": {
"version": "1.1.2",
@@ -4310,7 +4314,8 @@
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
- "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
+ "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=",
+ "optional": true
},
"fabric": {
"version": "2.4.5",
@@ -4629,7 +4634,8 @@
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
- "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+ "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+ "optional": true
},
"aproba": {
"version": "1.2.0",
@@ -4650,12 +4656,14 @@
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
- "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+ "optional": true
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+ "optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@@ -4670,17 +4678,20 @@
"code-point-at": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
- "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
+ "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
+ "optional": true
},
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
- "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+ "optional": true
},
"console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
- "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
+ "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
+ "optional": true
},
"core-util-is": {
"version": "1.0.2",
@@ -4797,7 +4808,8 @@
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
- "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
+ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
+ "optional": true
},
"ini": {
"version": "1.3.5",
@@ -4809,6 +4821,7 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
+ "optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@@ -4823,6 +4836,7 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+ "optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@@ -4830,12 +4844,14 @@
"minimist": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
- "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
+ "optional": true
},
"minipass": {
"version": "2.2.4",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.2.4.tgz",
"integrity": "sha512-hzXIWWet/BzWhYs2b+u7dRHlruXhwdgvlTMDKC6Cb1U7ps6Ac6yQlR39xsbjWJE377YTCtKwIXIpJ5oP+j5y8g==",
+ "optional": true,
"requires": {
"safe-buffer": "^5.1.1",
"yallist": "^3.0.0"
@@ -4854,6 +4870,7 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+ "optional": true,
"requires": {
"minimist": "0.0.8"
}
@@ -4934,7 +4951,8 @@
"number-is-nan": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
- "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
+ "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
+ "optional": true
},
"object-assign": {
"version": "4.1.1",
@@ -4946,6 +4964,7 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+ "optional": true,
"requires": {
"wrappy": "1"
}
@@ -5031,7 +5050,8 @@
"safe-buffer": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
- "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
+ "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==",
+ "optional": true
},
"safer-buffer": {
"version": "2.1.2",
@@ -5067,6 +5087,7 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
+ "optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@@ -5086,6 +5107,7 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+ "optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@@ -5129,12 +5151,14 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
- "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+ "optional": true
},
"yallist": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz",
- "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k="
+ "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=",
+ "optional": true
}
}
},
@@ -6213,7 +6237,8 @@
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
- "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM="
+ "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
+ "optional": true
},
"jsdom": {
"version": "9.12.0",
@@ -13369,7 +13394,8 @@
"psl": {
"version": "1.1.31",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz",
- "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw=="
+ "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==",
+ "optional": true
},
"public-encrypt": {
"version": "4.0.3",
@@ -14969,7 +14995,8 @@
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
- "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
+ "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
+ "optional": true
},
"type-check": {
"version": "0.3.2",
@@ -15749,6 +15776,7 @@
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz",
"integrity": "sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw==",
+ "optional": true,
"requires": {
"iconv-lite": "0.4.24"
}