You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by op...@apache.org on 2021/12/09 01:37:06 UTC
[iceberg] branch master updated: Aliyun: Add OSS integration test rule (#3687)
This is an automated email from the ASF dual-hosted git repository.
openinx pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new c972a34 Aliyun: Add OSS integration test rule (#3687)
c972a34 is described below
commit c972a344ca415f0548a2847c334fea322f51c82c
Author: openinx <op...@gmail.com>
AuthorDate: Thu Dec 9 09:36:55 2021 +0800
Aliyun: Add OSS integration test rule (#3687)
---
.../AliyunOSSTestUtility.java => TestUtility.java} | 55 ++++++++--
.../iceberg/aliyun/oss/AliyunOSSTestBase.java | 3 +-
.../iceberg/aliyun/oss/OSSIntegrationTestRule.java | 117 +++++++++++++++++++++
.../apache/iceberg/aliyun/oss/TestOSSFileIO.java | 4 +-
.../aliyun/oss/mock/AliyunOSSMockLocalStore.java | 3 +-
.../aliyun/oss/mock/TestLocalAliyunOSS.java | 11 +-
6 files changed, 182 insertions(+), 11 deletions(-)
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestUtility.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java
similarity index 53%
rename from aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestUtility.java
rename to aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java
index 0e872f3..762074d 100644
--- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestUtility.java
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java
@@ -17,25 +17,36 @@
* under the License.
*/
-package org.apache.iceberg.aliyun.oss;
+package org.apache.iceberg.aliyun;
+import org.apache.iceberg.aliyun.oss.AliyunOSSTestRule;
+import org.apache.iceberg.aliyun.oss.OSSURI;
import org.apache.iceberg.aliyun.oss.mock.AliyunOSSMockRule;
import org.apache.iceberg.common.DynConstructors;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class AliyunOSSTestUtility {
- private static final Logger LOG = LoggerFactory.getLogger(AliyunOSSTestUtility.class);
- private static final String ALIYUN_TEST_OSS_TEST_RULE_CLASS = "ALIYUN_TEST_OSS_TEST_RULE_CLASS";
+public class TestUtility {
+ private static final Logger LOG = LoggerFactory.getLogger(TestUtility.class);
- private AliyunOSSTestUtility() {
+ // System environment variables for Aliyun Access Key Pair.
+ private static final String ALIYUN_TEST_ACCESS_KEY_ID = "ALIYUN_TEST_ACCESS_KEY_ID";
+ private static final String ALIYUN_TEST_ACCESS_KEY_SECRET = "ALIYUN_TEST_ACCESS_KEY_SECRET";
+
+ // System environment variables for Aliyun OSS
+ private static final String ALIYUN_TEST_OSS_RULE_CLASS = "ALIYUN_TEST_OSS_TEST_RULE_CLASS";
+ private static final String ALIYUN_TEST_OSS_ENDPOINT = "ALIYUN_TEST_OSS_ENDPOINT";
+ private static final String ALIYUN_TEST_OSS_WAREHOUSE = "ALIYUN_TEST_OSS_WAREHOUSE";
+
+ private TestUtility() {
}
public static AliyunOSSTestRule initialize() {
AliyunOSSTestRule testRule;
- String implClass = System.getenv(ALIYUN_TEST_OSS_TEST_RULE_CLASS);
+ String implClass = System.getenv(ALIYUN_TEST_OSS_RULE_CLASS);
if (!Strings.isNullOrEmpty(implClass)) {
LOG.info("The initializing AliyunOSSTestRule implementation is: {}", implClass);
try {
@@ -56,4 +67,36 @@ public class AliyunOSSTestUtility {
return testRule;
}
+
+ public static String accessKeyId() {
+ return System.getenv(ALIYUN_TEST_ACCESS_KEY_ID);
+ }
+
+ public static String accessKeySecret() {
+ return System.getenv(ALIYUN_TEST_ACCESS_KEY_SECRET);
+ }
+
+ public static String ossEndpoint() {
+ return System.getenv(ALIYUN_TEST_OSS_ENDPOINT);
+ }
+
+ public static String ossWarehouse() {
+ return System.getenv(ALIYUN_TEST_OSS_WAREHOUSE);
+ }
+
+ public static String ossBucket() {
+ return ossWarehouseURI().bucket();
+ }
+
+ public static String ossKey() {
+ return ossWarehouseURI().key();
+ }
+
+ private static OSSURI ossWarehouseURI() {
+ String ossWarehouse = ossWarehouse();
+ Preconditions.checkNotNull(ossWarehouse,
+ "Please set a correct Aliyun OSS path for environment variable '%s'", ALIYUN_TEST_OSS_WAREHOUSE);
+
+ return new OSSURI(ossWarehouse);
+ }
}
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java
index 24e89bf..220a867 100644
--- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/AliyunOSSTestBase.java
@@ -20,6 +20,7 @@
package org.apache.iceberg.aliyun.oss;
import com.aliyun.oss.OSS;
+import org.apache.iceberg.aliyun.TestUtility;
import org.apache.iceberg.util.SerializableSupplier;
import org.junit.After;
import org.junit.Before;
@@ -27,7 +28,7 @@ import org.junit.ClassRule;
public abstract class AliyunOSSTestBase {
@ClassRule
- public static final AliyunOSSTestRule OSS_TEST_RULE = AliyunOSSTestUtility.initialize();
+ public static final AliyunOSSTestRule OSS_TEST_RULE = TestUtility.initialize();
private final SerializableSupplier<OSS> ossClient = OSS_TEST_RULE::createOSSClient;
private final String bucketName = OSS_TEST_RULE.testBucketName();
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/OSSIntegrationTestRule.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/OSSIntegrationTestRule.java
new file mode 100644
index 0000000..691d6d0
--- /dev/null
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/OSSIntegrationTestRule.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.aliyun.oss;
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.model.ListObjectsV2Request;
+import com.aliyun.oss.model.ListObjectsV2Result;
+import com.aliyun.oss.model.OSSObjectSummary;
+import org.apache.iceberg.aliyun.TestUtility;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+public class OSSIntegrationTestRule implements AliyunOSSTestRule {
+ // Aliyun access key pair.
+ private String accessKeyId;
+ private String accessKeySecret;
+
+ // Aliyun OSS configure values.
+ private String ossEndpoint;
+ private String ossBucket;
+ private String ossKey;
+
+ private volatile OSS lazyClient = null;
+
+ @Override
+ public String testBucketName() {
+ return ossBucket;
+ }
+
+ @Override
+ public String keyPrefix() {
+ return ossKey;
+ }
+
+ @Override
+ public void start() {
+ this.accessKeyId = TestUtility.accessKeyId();
+ this.accessKeySecret = TestUtility.accessKeySecret();
+
+ this.ossEndpoint = TestUtility.ossEndpoint();
+ this.ossBucket = TestUtility.ossBucket();
+ this.ossKey = TestUtility.ossKey();
+ }
+
+ @Override
+ public void stop() {
+ if (lazyClient != null) {
+ lazyClient.shutdown();
+ lazyClient = null;
+ }
+ }
+
+ @Override
+ public OSS createOSSClient() {
+ Preconditions.checkNotNull(ossEndpoint, "OSS endpoint cannot be null");
+ Preconditions.checkNotNull(accessKeyId, "OSS access key id cannot be null");
+ Preconditions.checkNotNull(accessKeySecret, "OSS access secret cannot be null");
+
+ return new OSSClientBuilder().build(ossEndpoint, accessKeyId, accessKeySecret);
+ }
+
+ @Override
+ public void setUpBucket(String bucket) {
+ Preconditions.checkArgument(
+ ossClient().doesBucketExist(bucket),
+ "Bucket %s does not exist, please create it firstly.", bucket);
+ }
+
+ @Override
+ public void tearDownBucket(String bucket) {
+ int maxKeys = 200;
+ String nextContinuationToken = null;
+ ListObjectsV2Result objectListingResult;
+ do {
+ ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request(bucket)
+ .withMaxKeys(maxKeys)
+ .withPrefix(ossKey)
+ .withContinuationToken(nextContinuationToken);
+ objectListingResult = ossClient().listObjectsV2(listObjectsV2Request);
+
+ for (OSSObjectSummary s : objectListingResult.getObjectSummaries()) {
+ ossClient().deleteObject(bucket, s.getKey());
+ }
+
+ nextContinuationToken = objectListingResult.getNextContinuationToken();
+ } while (objectListingResult.isTruncated());
+ }
+
+ private OSS ossClient() {
+ if (lazyClient == null) {
+ synchronized (OSSIntegrationTestRule.class) {
+ if (lazyClient == null) {
+ lazyClient = createOSSClient();
+ }
+ }
+ }
+
+ return lazyClient;
+ }
+}
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java
index 518d0b7..9bebfae 100644
--- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java
@@ -59,7 +59,9 @@ public class TestOSSFileIO extends AliyunOSSTestBase {
@After
public void afterFile() {
- fileIO.close();
+ if (fileIO != null) {
+ fileIO.close();
+ }
}
@Test
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java
index f9784a7..22bf1dd 100644
--- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/AliyunOSSMockLocalStore.java
@@ -35,6 +35,7 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.directory.api.util.Hex;
@@ -88,7 +89,7 @@ public class AliyunOSSMockLocalStore {
while ((numBytes = is.read(bytes)) != -1) {
md.update(bytes, 0, numBytes);
}
- return new String(Hex.encodeHex(md.digest()));
+ return new String(Hex.encodeHex(md.digest())).toUpperCase(Locale.ROOT);
}
private static void inputStreamToFile(InputStream inputStream, File targetFile) throws IOException {
diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/TestLocalAliyunOSS.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/TestLocalAliyunOSS.java
index 0ad5ebf..a2849f2 100644
--- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/TestLocalAliyunOSS.java
+++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/mock/TestLocalAliyunOSS.java
@@ -30,11 +30,12 @@ import java.io.InputStream;
import java.util.Objects;
import java.util.Random;
import java.util.UUID;
+import org.apache.iceberg.aliyun.TestUtility;
import org.apache.iceberg.aliyun.oss.AliyunOSSTestRule;
-import org.apache.iceberg.aliyun.oss.AliyunOSSTestUtility;
import org.apache.iceberg.relocated.com.google.common.io.ByteStreams;
import org.junit.After;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
@@ -42,7 +43,7 @@ import org.junit.Test;
public class TestLocalAliyunOSS {
@ClassRule
- public static final AliyunOSSTestRule OSS_TEST_RULE = AliyunOSSTestUtility.initialize();
+ public static final AliyunOSSTestRule OSS_TEST_RULE = TestUtility.initialize();
private final OSS oss = OSS_TEST_RULE.createOSSClient();
private final String bucketName = OSS_TEST_RULE.testBucketName();
@@ -69,6 +70,9 @@ public class TestLocalAliyunOSS {
@Test
public void testBuckets() {
+ Assume.assumeTrue("Aliyun integration test cannot delete existing bucket from test environment.",
+ OSS_TEST_RULE.getClass() == AliyunOSSMockRule.class);
+
Assert.assertTrue(doesBucketExist(bucketName));
assertThrows(() -> oss.createBucket(bucketName), OSSErrorCode.BUCKET_ALREADY_EXISTS);
@@ -81,6 +85,9 @@ public class TestLocalAliyunOSS {
@Test
public void testDeleteBucket() {
+ Assume.assumeTrue("Aliyun integration test cannot delete existing bucket from test environment.",
+ OSS_TEST_RULE.getClass() == AliyunOSSMockRule.class);
+
String bucketNotExist = String.format("bucket-not-existing-%s", UUID.randomUUID());
assertThrows(() -> oss.deleteBucket(bucketNotExist), OSSErrorCode.NO_SUCH_BUCKET);