You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/11/09 13:13:42 UTC
[systemds] 04/05: [MINOR] mvn parallel build
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 5f4566af4c8d20d862cd9671eabec26c9f6d00fc
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Tue Nov 8 15:47:47 2022 +0100
[MINOR] mvn parallel build
This commit modify the pom.xml and adds a configuration to mvn to enable
parallel building, and also adds a jvm config for mvn to reduce logging
when creating shaded jars.
all in all on my laptop build time for 'mvn clean package -P distribution'
is reduced from 70 sec to 55 sec (hot) and 50 sec (cold).
and if we do not do a clean build time is all the way down to 44 sec, from 65+.
While doing this i have moved antler to a profile, since we do not need
to run it every package since we include the antler generated code in
github.
This commit further fixes the compile for maven to reuse previous compiled classes
if no changes were made. Unfortunately it was broken because
of an empty file ColGroupDeltaDDC.java, that kept reporting stale for the
compiler since it did not produce a .class file, and therefore forced
the system to compile from scratch every execution.
Before to run a single test from command line there was a compile overhead
up to 40 sec, now it is reduced to 4 sec.
Change SkLearn name for package to install.
scikit-learn is the name of the python package not sklearn
---
.github/workflows/python.yml | 2 +-
.mvn/jvm.config | 1 +
.mvn/maven.config | 1 +
pom.xml | 79 +++++++++-------------
.../compress/colgroup/ColGroupDeltaDDC.java | 4 +-
5 files changed, 38 insertions(+), 49 deletions(-)
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index a968094b7b..d7abd26b9c 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -100,7 +100,7 @@ jobs:
pip install --upgrade pip
pip install --upgrade pip
pip install wheel
- pip install numpy py4j scipy sklearn requests pandas unittest-parallel
+ pip install numpy py4j scipy scikit-learn requests pandas unittest-parallel
- name: Build Python Package
run: |
diff --git a/.mvn/jvm.config b/.mvn/jvm.config
new file mode 100644
index 0000000000..cc81b2be4a
--- /dev/null
+++ b/.mvn/jvm.config
@@ -0,0 +1 @@
+-Dorg.slf4j.simpleLogger.showLogName=false -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade.mojo.ShadeMojo=off
diff --git a/.mvn/maven.config b/.mvn/maven.config
new file mode 100644
index 0000000000..ac0bd15dc2
--- /dev/null
+++ b/.mvn/maven.config
@@ -0,0 +1 @@
+-T 1.5C
diff --git a/pom.xml b/pom.xml
index 560db63520..b44160c700 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,6 +145,10 @@
</includes>
<targetPath>java/spoof</targetPath>
</resource>
+ <resource>
+ <directory>src/test/config/hadoop_bin_windows/bin</directory>
+ <targetPath>../lib/hadoop/bin</targetPath>
+ </resource>
</resources>
<plugins>
@@ -167,7 +171,6 @@
<type>jar</type>
<overWrite>true</overWrite>
<outputDirectory>${project.build.directory}/hadoop-test</outputDirectory>
- <includes>**/*</includes>
</artifactItem>
</artifactItems>
<overWriteReleases>false</overWriteReleases>
@@ -254,50 +257,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-resources</id>
- <phase>compile</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
- <resources>
- <resource>
- <directory>${basedir}/src/test/config/hadoop_bin_windows/bin</directory>
- <filtering>false</filtering>
- <includes>
- <include>*.*</include>
- </includes>
- </resource>
- </resources>
- <outputDirectory>${basedir}/target/lib/hadoop/bin</outputDirectory>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.antlr</groupId>
- <artifactId>antlr4-maven-plugin</artifactId>
- <configuration>
- <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
- <outputDirectory>${basedir}/src/main/java</outputDirectory>
- </configuration>
- <version>${antlr.version}</version>
- <executions>
- <execution>
- <id>antlr</id>
- <goals>
- <goal>antlr4</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
<plugin>
<!-- unit tests -->
<groupId>org.apache.maven.plugins</groupId>
@@ -310,8 +269,9 @@
<!-- 1C means the number of threads times 1 possible maximum forks for testing-->
<forkCount>${test-forkCount}</forkCount>
<reuseForks>false</reuseForks>
- <reportFormat>brief</reportFormat>
+ <!-- <reportFormat>brief</reportFormat> -->
<trimStackTrace>true</trimStackTrace>
+ <showSuccess>false</showSuccess>
<rerunFailingTestsCount>${rerun.failing.tests.count}</rerunFailingTestsCount>
</configuration>
</plugin>
@@ -589,6 +549,7 @@
<exclude>.gitmodules</exclude>
<exclude>.repository/</exclude>
<exclude>.idea/</exclude>
+ <exclude>.mvn/**</exclude>
<exclude>.git</exclude>
<exclude>.settings</exclude>
<exclude>.classpath</exclude>
@@ -693,6 +654,32 @@
</build>
</profile>
+ <profile>
+ <id>antlr</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr4-maven-plugin</artifactId>
+ <configuration>
+ <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
+ <outputDirectory>${basedir}/src/main/java</outputDirectory>
+ </configuration>
+ <version>${antlr.version}</version>
+ <executions>
+ <execution>
+ <id>antlr</id>
+ <goals>
+ <goal>antlr4</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ </profile>
+
<profile>
<!-- Profile to create binary distributions. Execute with `mvn clean package
-P distribution` -->
diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java
index 2ec72f7cca..2666860ca6 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupDeltaDDC.java
@@ -23,7 +23,7 @@ package org.apache.sysds.runtime.compress.colgroup;
* Class to encapsulate information about a column group that is first delta encoded then encoded with dense dictionary
* encoding (DeltaDDC).
*/
-// public class ColGroupDeltaDDC extends ColGroupDDC {
+public class ColGroupDeltaDDC { // extends ColGroupDDC
// private static final long serialVersionUID = -1045556313148564147L;
@@ -77,4 +77,4 @@ package org.apache.sysds.runtime.compress.colgroup;
// public AColGroup scalarOperation(ScalarOperator op) {
// return new ColGroupDeltaDDC(_colIndexes, _dict.applyScalarOp(op), _data, getCachedCounts());
// }
-// }
+}