You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sn...@apache.org on 2019/01/31 22:43:20 UTC
[incubator-pinot] branch master updated: Remove dependencies from
libraries under category X licenses. (#3718)
This is an automated email from the ASF dual-hosted git repository.
snlee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7e28022 Remove dependencies from libraries under category X licenses. (#3718)
7e28022 is described below
commit 7e28022d5b4f412e57a098d0cfc3fedb8e27cd3d
Author: Seunghyun Lee <sn...@linkedin.com>
AuthorDate: Thu Jan 31 14:43:14 2019 -0800
Remove dependencies from libraries under category X licenses. (#3718)
For an ASF project, its distribution should not contain any of work under
category X license. After running "mvn project-info-reports:dependencies",
three libraries are under category X libraries for our distribution package.
This PR resolves this issue.
1. Removed "pinot-perf" module from distribution package because pinot-perf
module is pulling "jmh-core" based on GPL 2 license, which is a category X.
2. Fixed pom file to generate *.sh files for benchmarks in "pinot-perf" and
added on documentation on how to build and run.
3. Bumped up the version of org.testng from 6.0.1 to 6.11 because the old
version was pulling in "org.beanshell:bsh:2.0b4", which is also based on
LGPL license. The newer version no longer pulls beanshell.
4. Bumped up the swagger version from 1.5.10 to 1.5.16. The old version was
pulling "org.reflections:reflections:0.9.10", which pulls
"com.google.code.findbugs:annotations", which was based on LGPL.
For the list of category X licenses, please refer
https://www.apache.org/legal/resolved.html
---
.../selection/SelectionOperatorServiceTest.java | 22 ++++----
pinot-distribution/pom.xml | 4 --
...umSegmentAssignmentStrategyIntegrationTest.java | 14 ++---
pinot-perf/README.md | 20 +++++++
pinot-perf/pom.xml | 66 +++++++++++++++++++---
pom.xml | 16 +-----
6 files changed, 100 insertions(+), 42 deletions(-)
diff --git a/pinot-core/src/test/java/org/apache/pinot/query/selection/SelectionOperatorServiceTest.java b/pinot-core/src/test/java/org/apache/pinot/query/selection/SelectionOperatorServiceTest.java
index 55092d7..9f83089 100644
--- a/pinot-core/src/test/java/org/apache/pinot/query/selection/SelectionOperatorServiceTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/query/selection/SelectionOperatorServiceTest.java
@@ -135,8 +135,9 @@ public class SelectionOperatorServiceTest {
{0L, 1.0, 2.0, 3.0, "4", new long[]{5L}, new double[]{6.0}, new double[]{7.0}, new double[]{8.0}, new String[]{"9"}};
Serializable[] expectedCompatibleRow1 =
{1L, 2.0, 3.0, 4.0, "5", new long[]{6L}, new double[]{7.0}, new double[]{8.0}, new double[]{9.0}, new String[]{"10"}};
- Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 0), expectedRow1);
- Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 1), expectedCompatibleRow1);
+ Assert.assertTrue(Arrays.deepEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 0), expectedRow1));
+ Assert.assertTrue(
+ Arrays.deepEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 1), expectedCompatibleRow1));
}
@Test
@@ -153,8 +154,9 @@ public class SelectionOperatorServiceTest {
{0, 1L, 2.0F, 3.0, "4", new int[]{5}, new long[]{6L}, new float[]{7.0F}, new double[]{8.0}, new String[]{"9"}};
Serializable[] expectedRow2 =
{1L, 2.0F, 3.0, 4, "5", new long[]{6L}, new float[]{7.0F}, new double[]{8.0}, new int[]{9}, new String[]{"10"}};
- Assert.assertEquals(resultRows.get(0), expectedRow1);
- Assert.assertEquals(resultRows.get(1), expectedRow2);
+
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(0), expectedRow1));
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(1), expectedRow2));
int[] columnIndices =
SelectionOperatorUtils.getColumnIndicesWithoutOrdering(selectionResults.getColumns(), _dataSchema);
@@ -166,8 +168,8 @@ public class SelectionOperatorServiceTest {
{"0", "1.0", "2.0", "3.0", "4", new String[]{"5"}, new String[]{"6.0"}, new String[]{"7.0"}, new String[]{"8.0"}, new String[]{"9"}};
Serializable[] expectedFormattedRow2 =
{"1", "2.0", "3.0", "4.0", "5", new String[]{"6"}, new String[]{"7.0"}, new String[]{"8.0"}, new String[]{"9.0"}, new String[]{"10"}};
- Assert.assertEquals(resultRows.get(0), expectedFormattedRow1);
- Assert.assertEquals(resultRows.get(1), expectedFormattedRow2);
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(0), expectedFormattedRow1));
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(1), expectedFormattedRow2));
}
@Test
@@ -186,8 +188,8 @@ public class SelectionOperatorServiceTest {
{1L, 2.0F, 3.0, 4, "5", new long[]{6L}, new float[]{7.0F}, new double[]{8.0}, new int[]{9}, new String[]{"10"}};
Serializable[] expectedRow2 =
{0, 1L, 2.0F, 3.0, "4", new int[]{5}, new long[]{6L}, new float[]{7.0F}, new double[]{8.0}, new String[]{"9"}};
- Assert.assertEquals(resultRows.get(0), expectedRow1);
- Assert.assertEquals(resultRows.get(1), expectedRow2);
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(0), expectedRow1));
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(1), expectedRow2));
int[] columnIndices =
SelectionOperatorUtils.getColumnIndicesWithOrdering(selectionResults.getColumns(), _dataSchema);
@@ -197,7 +199,7 @@ public class SelectionOperatorServiceTest {
{"1", "2.0", "3.0", "4.0", "5", new String[]{"6"}, new String[]{"7.0"}, new String[]{"8.0"}, new String[]{"9.0"}, new String[]{"10"}};
Serializable[] expectedFormattedRow2 =
{"0", "1.0", "2.0", "3.0", "4", new String[]{"5"}, new String[]{"6.0"}, new String[]{"7.0"}, new String[]{"8.0"}, new String[]{"9"}};
- Assert.assertEquals(resultRows.get(0), expectedFormattedRow1);
- Assert.assertEquals(resultRows.get(1), expectedFormattedRow2);
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(0), expectedFormattedRow1));
+ Assert.assertTrue(Arrays.deepEquals(resultRows.get(1), expectedFormattedRow2));
}
}
diff --git a/pinot-distribution/pom.xml b/pinot-distribution/pom.xml
index f5dfa34..1dc17ff 100644
--- a/pinot-distribution/pom.xml
+++ b/pinot-distribution/pom.xml
@@ -84,10 +84,6 @@
<groupId>org.apache.pinot</groupId>
<artifactId>pinot-tools</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.pinot</groupId>
- <artifactId>pinot-perf</artifactId>
- </dependency>
</dependencies>
<build>
<plugins>
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BalanceNumSegmentAssignmentStrategyIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BalanceNumSegmentAssignmentStrategyIntegrationTest.java
index 0c04d74..b8ac99a 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BalanceNumSegmentAssignmentStrategyIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BalanceNumSegmentAssignmentStrategyIntegrationTest.java
@@ -58,7 +58,13 @@ public class BalanceNumSegmentAssignmentStrategyIntegrationTest extends UploadRe
}
}
- @Test(dataProvider = "configProvider")
+ @DataProvider(name = "tableNameProvider")
+ public Object[][] configProvider() {
+ Object[][] configs = {{"disabledInstancesTable", SegmentVersion.v3}};
+ return configs;
+ }
+
+ @Test(dataProvider = "tableNameProvider")
public void testSegmentAssignmentStrategy(String tableName, SegmentVersion version)
throws Exception {
// Upload nine segments
@@ -91,12 +97,6 @@ public class BalanceNumSegmentAssignmentStrategyIntegrationTest extends UploadRe
return segmentsPerInstance;
}
- @DataProvider(name = "tableNameProvider")
- public Object[][] configProvider() {
- Object[][] configs = {{"disabledInstancesTable", SegmentVersion.v3}};
- return configs;
- }
-
@Test(dataProvider = "tableNameProvider")
public void testNoAssignmentToDisabledInstances(String tableName, SegmentVersion version)
throws Exception {
diff --git a/pinot-perf/README.md b/pinot-perf/README.md
new file mode 100644
index 0000000..1f0e95e
--- /dev/null
+++ b/pinot-perf/README.md
@@ -0,0 +1,20 @@
+Pint Perf Package
+===
+Pinot perf package contains a set of performance benchmark for Pinot components.
+
+Note: this package will pull `org.openjdk.jmh:jmh-core`, which is based on `GPL 2 license`.
+
+# Steps for running benchmark
+
+1. Build the source
+```
+$ cd <root_source_code>/pinot-perf
+$ mvn package -DskipTests
+```
+2. The above cmd will generate `target/pinot-perf-pkg`
+
+3. Run benchmark using generated scripts
+```
+$ cd target/pinot-perf-pkg/bin
+$ ./pinot-BenchmarkDictionary.sh
+```
\ No newline at end of file
diff --git a/pinot-perf/pom.xml b/pinot-perf/pom.xml
index 7383731..6a08979 100644
--- a/pinot-perf/pom.xml
+++ b/pinot-perf/pom.xml
@@ -93,6 +93,7 @@
<dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-core</artifactId>
+ <version>1.15</version>
<exclusions>
<exclusion>
<groupId>net.sf.jopt-simple</groupId>
@@ -101,14 +102,15 @@
</exclusions>
</dependency>
<dependency>
- <groupId>net.sf.jopt-simple</groupId>
- <artifactId>jopt-simple</artifactId>
- </dependency>
- <dependency>
<groupId>org.openjdk.jmh</groupId>
<artifactId>jmh-generator-annprocess</artifactId>
+ <version>1.15</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
@@ -118,12 +120,60 @@
<configuration>
<programs>
<program>
- <mainClass>org.apache.pinot.perf.PerfBenchmarkTest</mainClass>
- <name>pinot-perfBenchmarkTest</name>
+ <mainClass>org.apache.pinot.perf.BenchmarkDictionary</mainClass>
+ <name>pinot-BenchmarkDictionary</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkDictionaryCreation</mainClass>
+ <name>pinot-BenchmarkDictionaryCreation</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkFixedIntArrayOffHeapIdMap</mainClass>
+ <name>pinot-BenchmarkFixedIntArrayOffHeapIdMap</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkOffHeapDictionaryMemory</mainClass>
+ <name>pinot-BenchmarkOffHeapDictionaryMemory</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkOfflineIndexReader</mainClass>
+ <name>pinot-BenchmarkOfflineIndexReader</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkOrDocIdIterator</mainClass>
+ <name>pinot-BenchmarkOrDocIdIterator</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkQueryEngine</mainClass>
+ <name>pinot-BenchmarkQueryEngine</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkRealtimeConsumptionSpeed</mainClass>
+ <name>pinot-BenchmarkRealtimeConsumptionSpeed</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.BenchmarkStringDictionary</mainClass>
+ <name>pinot-BenchmarkStringDictionary</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.DictionaryDumper</mainClass>
+ <name>pinot-DictionaryDumper</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.ForwardIndexWriterBenchmark</mainClass>
+ <name>pinot-ForwardIndexWriterBenchmark</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.RawIndexBenchmark</mainClass>
+ <name>pinot-RawIndexBenchmark</name>
+ </program>
+ <program>
+ <mainClass>org.apache.pinot.perf.RealtimeStressTest</mainClass>
+ <name>pinot-RealtimeStressTest</name>
</program>
<program>
- <mainClass>org.apache.pinot.perf.PerfBenchmarkRunner</mainClass>
- <name>pinot-perfBenchmarkRunner</name>
+ <mainClass>org.apache.pinot.perf.StringDictionaryPerfTest</mainClass>
+ <name>pinot-StringDictionaryPerfTest</name>
</program>
</programs>
<repositoryLayout>flat</repositoryLayout>
diff --git a/pom.xml b/pom.xml
index f7f707c..bf2c4c3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,7 +114,7 @@
<jackson.version>2.9.8</jackson.version>
<async-http-client.version>1.9.21</async-http-client.version>
<jersey.version>2.28</jersey.version>
- <swagger.version>1.5.10</swagger.version>
+ <swagger.version>1.5.16</swagger.version>
<hadoop.version>2.7.0</hadoop.version>
<antlr.version>4.6</antlr.version>
<!-- commons-configuration, hadoop-common, hadoop-client use commons-lang -->
@@ -284,7 +284,7 @@
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
- <version>6.0.1</version>
+ <version>6.11</version>
</dependency>
<dependency>
<groupId>me.lemire.integercompression</groupId>
@@ -304,7 +304,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>19.0</version>
+ <version>20.0</version>
</dependency>
<dependency>
<groupId>com.microsoft.azure</groupId>
@@ -711,16 +711,6 @@
<version>3.0.12</version>
</dependency>
<dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-generator-annprocess</artifactId>
- <version>1.8</version>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-core</artifactId>
- <version>1.15</version>
- </dependency>
- <dependency>
<groupId>args4j</groupId>
<artifactId>args4j</artifactId>
<version>2.32</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org