You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ma...@apache.org on 2020/04/23 21:35:14 UTC
[incubator-pinot] branch s3-fix updated (35b00ad -> 5c6fe7e)
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a change to branch s3-fix
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
omit 35b00ad Disable S3MockListner in S3PinotFSTest.
add 410fd70 Use 8byte offsets in chunk based raw index creator (#5285)
add 62799a3 Improve GroupBy query performance by setting initial size for OpenHashMap used in DictionaryBasedGroupKeyGenerator. (#5291)
add 7fd2110 Disable S3MockListner in S3PinotFSTest. (#5292)
new 5c6fe7e Fix
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (35b00ad)
\
N -- N -- N refs/heads/s3-fix (5c6fe7e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../reader/impl/v1/BaseChunkSingleValueReader.java | 27 +++++---
.../impl/v1/VarByteChunkSingleValueReader.java | 6 +-
.../writer/impl/v1/BaseChunkSingleValueWriter.java | 30 +++++++--
.../impl/v1/FixedByteChunkSingleValueWriter.java | 5 +-
.../impl/v1/VarByteChunkSingleValueWriter.java | 8 ++-
.../groupby/DictionaryBasedGroupKeyGenerator.java | 21 ++++---
.../segment/memory/PinotNativeOrderLBuffer.java | 2 +-
.../segment/memory/PinotNonNativeOrderLBuffer.java | 2 +-
.../FixedByteChunkSingleValueReaderWriteTest.java | 25 +++++---
.../VarByteChunkSingleValueReaderWriteTest.java | 68 ++++++++++++++++-----
.../src/test/resources/data/fixedByteCompressed.v2 | Bin 0 -> 8098 bytes
pinot-core/src/test/resources/data/fixedByteRaw.v2 | Bin 0 -> 16036 bytes
.../resources/data/varByteStringsCompressed.v2 | Bin 0 -> 17355 bytes
.../src/test/resources/data/varByteStringsRaw.v2 | Bin 0 -> 286902 bytes
.../pinot/plugin/filesystem/PinotS3Mock.java} | 34 +++++++----
.../pinot/plugin/filesystem/S3PinotFSTest.java | 54 ++++++++--------
16 files changed, 187 insertions(+), 95 deletions(-)
create mode 100644 pinot-core/src/test/resources/data/fixedByteCompressed.v2
create mode 100644 pinot-core/src/test/resources/data/fixedByteRaw.v2
create mode 100644 pinot-core/src/test/resources/data/varByteStringsCompressed.v2
create mode 100644 pinot-core/src/test/resources/data/varByteStringsRaw.v2
copy pinot-plugins/{pinot-input-format/pinot-thrift/src/main/java/org/apache/pinot/plugin/inputformat/thrift/ThriftRecordExtractorConfig.java => pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/PinotS3Mock.java} (60%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Fix
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch s3-fix
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5c6fe7e5e80040143bb74b55d349ea8dc4f4eaae
Author: Mayank Shrivastava <ms...@linkedin.com>
AuthorDate: Thu Apr 23 14:34:39 2020 -0700
Fix
---
.../pinot/plugin/filesystem/PinotS3Mock.java | 49 ++++++++++++++++++++
.../pinot/plugin/filesystem/S3PinotFSTest.java | 54 +++++++++++-----------
2 files changed, 75 insertions(+), 28 deletions(-)
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/PinotS3Mock.java b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/PinotS3Mock.java
new file mode 100644
index 0000000..99bd13a
--- /dev/null
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/PinotS3Mock.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.plugin.filesystem;
+
+import com.adobe.testing.s3mock.testsupport.common.S3MockStarter;
+import java.util.Map;
+
+
+public class PinotS3Mock extends S3MockStarter {
+
+ private PinotS3Mock() {
+ super((Map) null);
+ }
+
+ private PinotS3Mock(Map<String, Object> properties) {
+ super(properties);
+ }
+
+ void bootstrap() {
+ this.start();
+ }
+
+ void terminate() {
+ this.stop();
+ }
+
+ public static class PinotS3MockBuilder extends BaseBuilder<PinotS3Mock> {
+ @Override
+ public PinotS3Mock build() {
+ return new PinotS3Mock(arguments);
+ }
+ }
+}
\ No newline at end of file
diff --git a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
index 421ea68..182c281 100644
--- a/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
+++ b/pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java
@@ -18,9 +18,7 @@
*/
package org.apache.pinot.plugin.filesystem;
-import com.adobe.testing.s3mock.testng.S3Mock;
import java.io.File;
-import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
@@ -34,37 +32,32 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Object;
-@Test(enabled = false)
-//@Listeners(com.adobe.testing.s3mock.testng.S3MockListener.class)
+@Test
public class S3PinotFSTest {
final String DELIMITER = "/";
S3PinotFS _s3PinotFS;
S3Client _s3Client;
+ PinotS3Mock _s3Mock;
final String BUCKET = "test-bucket";
final String SCHEME = "s3";
final String FILE_FORMAT = "%s://%s/%s";
final String DIR_FORMAT = "%s://%s";
+ final Integer HTTP_PORT = 9452;
+ final Integer HTTPS_PORT = 9453;
@BeforeClass
public void setUp() {
- S3Mock s3Mock = S3Mock.getInstance();
- _s3Client = s3Mock.createS3ClientV2();
+ _s3Mock = new PinotS3Mock.PinotS3MockBuilder().withHttpPort(HTTP_PORT).withHttpsPort(HTTPS_PORT)
+ .withInitialBuckets(BUCKET).build();
+ _s3Mock.bootstrap();
+ _s3Client = _s3Mock.createS3ClientV2();
_s3PinotFS = new S3PinotFS();
_s3PinotFS.init(_s3Client);
- _s3Client.createBucket(CreateBucketRequest.builder().bucket(BUCKET).build());
- }
-
- @AfterClass
- public void tearDown()
- throws IOException {
- _s3PinotFS.close();
- _s3Client.close();
}
private void createEmptyFile(String folderName, String fileName) {
@@ -73,7 +66,7 @@ public class S3PinotFSTest {
.putObject(S3TestUtils.getPutObjectRequest(BUCKET, fileNameWithFolder), RequestBody.fromBytes(new byte[0]));
}
- @Test(enabled = false)
+ @Test
public void testTouchFileInBucket()
throws Exception {
@@ -92,7 +85,7 @@ public class S3PinotFSTest {
Assert.assertTrue(Arrays.equals(response, originalFiles));
}
- @Test(enabled = false)
+ @Test
public void testTouchFilesInFolder()
throws Exception {
@@ -113,7 +106,7 @@ public class S3PinotFSTest {
Assert.assertTrue(Arrays.equals(response, Arrays.stream(originalFiles).map(x -> folder + DELIMITER + x).toArray()));
}
- @Test(enabled = false)
+ @Test
public void testListFilesInBucketNonRecursive()
throws Exception {
String[] originalFiles = new String[]{"a-list.txt", "b-list.txt", "c-list.txt"};
@@ -130,7 +123,7 @@ public class S3PinotFSTest {
Assert.assertTrue(Arrays.equals(actualFiles, originalFiles));
}
- @Test(enabled = false)
+ @Test
public void testListFilesInFolderNonRecursive()
throws Exception {
String folder = "list-files";
@@ -148,7 +141,7 @@ public class S3PinotFSTest {
Arrays.equals(Arrays.stream(originalFiles).map(x -> folder + DELIMITER + x).toArray(), actualFiles));
}
- @Test(enabled = false)
+ @Test
public void testListFilesInFolderRecursive()
throws Exception {
String folder = "list-files-rec";
@@ -170,7 +163,7 @@ public class S3PinotFSTest {
Assert.assertTrue(Arrays.equals(expectedResultList.toArray(), actualFiles));
}
- @Test(enabled = false)
+ @Test
public void testDeleteFile()
throws Exception {
String[] originalFiles = new String[]{"a-delete.txt", "b-delete.txt", "c-delete.txt"};
@@ -196,7 +189,7 @@ public class S3PinotFSTest {
Assert.assertTrue(Arrays.equals(actualResponse, expectedResultList.toArray()));
}
- @Test(enabled = false)
+ @Test
public void testDeleteFolder()
throws Exception {
String[] originalFiles = new String[]{"a-delete-2.txt", "b-delete-2.txt", "c-delete-2.txt"};
@@ -217,7 +210,7 @@ public class S3PinotFSTest {
Assert.assertEquals(0, actualResponse.length);
}
- @Test(enabled = false)
+ @Test
public void testIsDirectory()
throws Exception {
String[] originalFiles = new String[]{"a-dir.txt", "b-dir.txt", "c-dir.txt"};
@@ -241,7 +234,7 @@ public class S3PinotFSTest {
Assert.assertFalse(notIsDir);
}
- @Test(enabled = false)
+ @Test
public void testExists()
throws Exception {
String[] originalFiles = new String[]{"a-ex.txt", "b-ex.txt", "c-ex.txt"};
@@ -269,7 +262,7 @@ public class S3PinotFSTest {
Assert.assertFalse(fileNotExists);
}
- @Test(enabled = false)
+ @Test
public void testCopyFromAndToLocal()
throws Exception {
String fileName = "copyFile.txt";
@@ -289,7 +282,7 @@ public class S3PinotFSTest {
fileToDownload.deleteOnExit();
}
- @Test(enabled = false)
+ @Test
public void testOpenFile()
throws Exception {
String fileName = "sample.txt";
@@ -302,7 +295,7 @@ public class S3PinotFSTest {
Assert.assertEquals(actualContents, fileContent);
}
- @Test(enabled = false)
+ @Test
public void testMkdir()
throws Exception {
String folderName = "my-test-folder";
@@ -312,4 +305,9 @@ public class S3PinotFSTest {
HeadObjectResponse headObjectResponse = _s3Client.headObject(S3TestUtils.getHeadObjectRequest(BUCKET, folderName));
Assert.assertTrue(headObjectResponse.sdkHttpResponse().isSuccessful());
}
-}
+
+ @AfterClass
+ public void tearDown() {
+ _s3Mock.terminate();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org