[GitHub] [hadoop-ozone] ChenSammi commented on a change in pull request #1296: HDDS-4053. Volume space: add quotaUsageInBytes and update it when write and delete key.

[GitBox <gi...@apache.org>]

ChenSammi commented on a change in pull request #1296:
URL: https://github.com/apache/hadoop-ozone/pull/1296#discussion_r477015608



##########
File path: hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
##########
@@ -79,6 +79,8 @@
 
   private int listCacheSize;
 
+  private long quotaUsageInBytes;

Review comment:
       quotaUsageInBytes -> usedBytes 

##########
File path: hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
##########
@@ -353,6 +353,7 @@ message VolumeInfo {
     optional uint64 objectID = 8;
     optional uint64 updateID = 9;
     optional uint64 modificationTime = 10;
+    optional uint64 quotaUsageInBytes = 11;

Review comment:
       quotaUsageInBytes -> usedBytes

##########
File path: hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
##########
@@ -703,6 +707,250 @@ public void testPutKey() throws IOException {
     }
   }
 
+  @Test
+  public void testVolumeQuotaUsageInBytes() throws IOException {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    OzoneVolume volume = null;
+
+    int blockSize = (int) ozoneManager.getConfiguration().getStorageSize(
+        OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+
+    // Write data larger than one block size.
+    String value = generateData(blockSize + 100,
+        (byte) RandomUtils.nextLong()).toString();
+
+    int valueLength = value.getBytes().length;
+    long currentQuotaUsage = 0L;
+    store.createVolume(volumeName);
+    volume = store.getVolume(volumeName);
+    // The initial value should be 0
+    Assert.assertEquals(0L, volume.getQuotaUsageInBytes());
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+
+    //Case1: Test the volumeQuotaUsageInBytes of ONE replications.
+    String keyName1 = UUID.randomUUID().toString();
+    writeKey(bucket, keyName1, ONE, value, valueLength);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength, volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength;
+
+    // Case2: Test overwrite the same KeyName under ONE Replicates, the
+    // keyLocationVersions of the Key is 2.
+    String keyName2 = UUID.randomUUID().toString();
+    writeKey(bucket, keyName2, ONE, value, valueLength);
+    // Overwrite the keyName2
+    writeKey(bucket, keyName2, ONE, value, valueLength);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 2;
+
+    // Case3: Test the volumeQuotaUsageInBytes of THREE replications.
+    String keyName3 = UUID.randomUUID().toString();
+    writeKey(bucket, keyName3, THREE, value, valueLength);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 3;
+
+    // Case4: Test overwrite the same KeyName under THREE Replicates, the
+    // keyLocationVersions of the Key is 2.
+    String keyName4 = UUID.randomUUID().toString();
+    writeKey(bucket, keyName4, THREE, value, valueLength);
+    // Overwrite the keyName4
+    writeKey(bucket, keyName4, THREE, value, valueLength);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 3 * 2;
+
+    //Case5: Do not specify the value Length, simulate HDFS api writing.
+    // Test the volumeQuotaUsageInBytes of ONE replications.
+    String keyName5 = UUID.randomUUID().toString();
+    writeFile(bucket, keyName5, ONE, value, 0);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength;
+
+    // Case6: Do not specify the value Length, simulate HDFS api writing.
+    // Test overwrite the same KeyName under ONE Replicates, the
+    // keyLocationVersions of the Key is 2.
+    String keyName6 = UUID.randomUUID().toString();
+    writeFile(bucket, keyName6, ONE, value, 0);
+    // Overwrite the keyName6
+    writeFile(bucket, keyName6, ONE, value, 0);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 2 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 2;
+
+    // Case7: Do not specify the value Length, simulate HDFS api writing.
+    // Test the volumeQuotaUsageInBytes of THREE replications.
+    String keyName7 = UUID.randomUUID().toString();
+    writeFile(bucket, keyName7, THREE, value, 0);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 3 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 3;
+
+    // Case8: Do not specify the value Length, simulate HDFS api writing.
+    // Test overwrite the same KeyName under THREE Replicates, the
+    // keyLocationVersions of the Key is 2.
+    String keyName8 = UUID.randomUUID().toString();
+    writeFile(bucket, keyName8, THREE, value, 0);
+    // Overwrite the keyName8
+    writeFile(bucket, keyName8, THREE, value, 0);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(valueLength * 3 * 2 + currentQuotaUsage,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage += valueLength * 3 * 2;
+
+    // Case9: Test volumeQuotaUsageInBytes when delete key of ONE replications.
+    bucket.deleteKey(keyName1);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(currentQuotaUsage - valueLength,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage -= valueLength;
+
+    // Case10: Test volumeQuotaUsageInBytes when delete key of THREE
+    // replications.
+    bucket.deleteKey(keyName3);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(currentQuotaUsage - valueLength * 3,
+        volume.getQuotaUsageInBytes());
+    currentQuotaUsage -= valueLength * 3;
+
+    // Case11: Test volumeQuotaUsageInBytes when Test Delete keys. At this
+    // point all keys are deleted, volumeQuotaUsageInBytes should be 0
+    List<String> keyList = new ArrayList<>();
+    keyList.add(keyName2);
+    keyList.add(keyName4);
+    keyList.add(keyName5);
+    keyList.add(keyName6);
+    keyList.add(keyName7);
+    keyList.add(keyName8);
+    bucket.deleteKeys(keyList);
+    volume = store.getVolume(volumeName);
+    Assert.assertEquals(0, volume.getQuotaUsageInBytes());
+  }
+
+  @Test
+  public void testVolumeQuotaWithMultiThread() throws IOException,
+      InterruptedException{
+    String volumeName = UUID.randomUUID().toString();
+
+    int blockSize = (int) ozoneManager.getConfiguration().getStorageSize(
+        OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+    // Write data larger than one block size.
+    String value = generateData(blockSize + 100,
+        (byte) RandomUtils.nextLong()).toString();
+
+    int valueLength = value.getBytes().length;
+    long currentQuotaUsage = 0L;
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    // The initial value should be 0
+    Assert.assertEquals(0L, volume.getQuotaUsageInBytes());
+
+    CountDownLatch latch = new CountDownLatch(2);
+    AtomicInteger failCount = new AtomicInteger(0);
+
+    // Multiple threads write different buckets and ensure that the volume
+    // quota is correct.
+    Runnable r = () -> {
+      try {
+        for (int i = 0; i < 10; i++) {
+          String keyName = UUID.randomUUID().toString();
+          String bucketName = UUID.randomUUID().toString();
+          volume.createBucket(bucketName);
+          OzoneBucket bucket = volume.getBucket(bucketName);
+          OzoneOutputStream out = bucket.createKey(keyName, valueLength,
+              STAND_ALONE, ONE, new HashMap<>());
+          out.write(value.getBytes());
+          out.close();
+        }
+        latch.countDown();
+      } catch (IOException ex) {
+        latch.countDown();
+        failCount.incrementAndGet();
+      }
+    };
+
+    Thread thread1 = new Thread(r);
+    Thread thread2 = new Thread(r);
+
+    thread1.start();
+    thread2.start();
+
+    latch.await(6000, TimeUnit.SECONDS);
+
+    if (failCount.get() > 0) {
+      fail("testVolumeQuotaWithMultiThread failed");
+    }
+    currentQuotaUsage += valueLength * 10 * 2;
+    Assert.assertEquals(currentQuotaUsage,
+        store.getVolume(volumeName).getQuotaUsageInBytes());
+
+  }
+
+  private void writeKey(OzoneBucket bucket, String keyName,
+      ReplicationFactor replication, String value, int valueLength)
+      throws IOException{
+    OzoneOutputStream out = bucket.createKey(keyName, valueLength, STAND_ALONE,
+        replication, new HashMap<>());
+    out.write(value.getBytes());
+    out.close();
+  }
+
+  private void writeFile(OzoneBucket bucket, String keyName,
+      ReplicationFactor replication, String value, int valueLength)
+      throws IOException{
+    OzoneOutputStream out = bucket.createFile(keyName, valueLength, STAND_ALONE,
+        replication, true, true);
+    out.write(value.getBytes());
+    out.close();
+  }
+
+  @Test
+  public void testVolumeQuotaWithUploadPart() throws IOException {
+    String volumeName = UUID.randomUUID().toString();
+    String bucketName = UUID.randomUUID().toString();
+    String keyName = UUID.randomUUID().toString();
+    int blockSize = (int) ozoneManager.getConfiguration().getStorageSize(
+        OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+    String sampleData = generateData(blockSize + 100,
+        (byte) RandomUtils.nextLong()).toString();
+    int valueLength = sampleData.getBytes().length;
+
+    store.createVolume(volumeName);
+    OzoneVolume volume = store.getVolume(volumeName);
+    // The initial value should be 0
+    Assert.assertEquals(0L, volume.getQuotaUsageInBytes());
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+    OmMultipartInfo multipartInfo = bucket.initiateMultipartUpload(keyName,
+        STAND_ALONE, ONE);
+
+    assertNotNull(multipartInfo);
+    String uploadID = multipartInfo.getUploadID();
+    Assert.assertEquals(volumeName, multipartInfo.getVolumeName());
+    Assert.assertEquals(bucketName, multipartInfo.getBucketName());
+    Assert.assertEquals(keyName, multipartInfo.getKeyName());
+    assertNotNull(multipartInfo.getUploadID());
+
+    OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
+        sampleData.length(), 1, uploadID);

Review comment:
       Can we upload more parts?

##########
File path: hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
##########
@@ -386,7 +408,7 @@ public OmVolumeArgs copyObject() {
     OmOzoneAclMap cloneAclMap = aclMap.copyObject();
 
     return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
-        quotaInCounts, cloneMetadata, cloneAclMap, creationTime,
-        modificationTime, objectID, updateID);
+        quotaInCounts, cloneMetadata, quotaUsageInBytes.sum(), cloneAclMap,
+        creationTime, modificationTime, objectID, updateID);
   }
 }

Review comment:
       Should getObjectInfo also be changed?

##########
File path: hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
##########
@@ -46,6 +47,7 @@
   private long quotaInBytes;
   private long quotaInCounts;
   private final OmOzoneAclMap aclMap;
+  private LongAdder quotaUsageInBytes = new LongAdder();

Review comment:
       add final 

##########
File path: hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
##########
@@ -172,6 +176,14 @@ public long getQuotaInCounts() {
   public OmOzoneAclMap getAclMap() {
     return aclMap;
   }
+
+  public LongAdder getQuotaUsageInBytes() {
+    return quotaUsageInBytes;
+  }
+
+  public void setQuotaUsageInBytes(long quotaUsageInBytes) {

Review comment:
       setQuotaUsageInBytes -> increase***




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-issues-help@hadoop.apache.org