You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by na...@apache.org on 2021/11/29 08:13:08 UTC
[ozone] branch HDDS-3630 updated: HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3630 by this push:
new cbecfe6 HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
cbecfe6 is described below
commit cbecfe62f9f0c2d0b66607581688fadf4c71f899
Author: Gui Hecheng <ma...@tencent.com>
AuthorDate: Mon Nov 29 16:12:47 2021 +0800
HDDS-5920. Allow to add ColumnFamilyOptions to tables. (#2790)
---
.../hdds/utils/db/DBColumnFamilyDefinition.java | 13 ++++
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 2 +-
.../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 77 ++++++++++++++++++++++
3 files changed, 91 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
index e1c4163..8c66bc6 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java
@@ -18,6 +18,8 @@
*/
package org.apache.hadoop.hdds.utils.db;
+import org.rocksdb.ColumnFamilyOptions;
+
import java.io.IOException;
/**
@@ -38,6 +40,8 @@ public class DBColumnFamilyDefinition<KEY, VALUE> {
private final Codec<VALUE> valueCodec;
+ private ColumnFamilyOptions cfOptions;
+
public DBColumnFamilyDefinition(
String tableName,
Class<KEY> keyType,
@@ -49,6 +53,7 @@ public class DBColumnFamilyDefinition<KEY, VALUE> {
this.keyCodec = keyCodec;
this.valueType = valueType;
this.valueCodec = valueCodec;
+ this.cfOptions = null;
}
public Table<KEY, VALUE> getTable(DBStore db) throws IOException {
@@ -78,4 +83,12 @@ public class DBColumnFamilyDefinition<KEY, VALUE> {
public Codec<VALUE> getValueCodec() {
return valueCodec;
}
+
+ public ColumnFamilyOptions getCfOptions() {
+ return this.cfOptions;
+ }
+
+ public void setCfOptions(ColumnFamilyOptions cfOptions) {
+ this.cfOptions = cfOptions;
+ }
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
index ad48a19..760a05c 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java
@@ -156,7 +156,7 @@ public final class DBStoreBuilder {
for (DBColumnFamilyDefinition columnFamily :
definition.getColumnFamilies()) {
- addTable(columnFamily.getName());
+ addTable(columnFamily.getName(), columnFamily.getCfOptions());
addCodec(columnFamily.getKeyType(), columnFamily.getKeyCodec());
addCodec(columnFamily.getValueType(), columnFamily.getValueCodec());
}
diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
index 99fcbae..ac4353f 100644
--- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
+++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestDBStoreBuilder.java
@@ -20,6 +20,8 @@
package org.apache.hadoop.hdds.utils.db;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.junit.Assert;
import org.junit.Before;
@@ -27,11 +29,14 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
+import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ColumnFamilyOptions;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
/**
* Tests RDBStore creation.
@@ -182,5 +187,77 @@ public class TestDBStoreBuilder {
}
}
+ @Test
+ public void builderWithColumnFamilyOptions() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ folder.newFolder().getAbsolutePath());
+ File newFolder = folder.newFolder();
+
+ if (!newFolder.exists()) {
+ Assert.assertTrue(newFolder.mkdirs());
+ }
+
+ String sampleTableName = "sampleTable";
+ final DBDefinition sampleDB = new DBDefinition() {
+
+ private final DBColumnFamilyDefinition<String, Long> sampleTable =
+ new DBColumnFamilyDefinition<>(sampleTableName,
+ String.class, new StringCodec(), Long.class, new LongCodec());
+ {
+ ColumnFamilyOptions cfOptions = new ColumnFamilyOptions();
+ // reverse the default option for check
+ cfOptions.setForceConsistencyChecks(
+ !cfOptions.forceConsistencyChecks());
+ sampleTable.setCfOptions(cfOptions);
+ }
+
+ @Override
+ public String getName() {
+ return "sampleDB";
+ }
+
+ @Override
+ public String getLocationConfigKey() {
+ return null;
+ }
+
+ @Override
+ public DBColumnFamilyDefinition[] getColumnFamilies() {
+ return new DBColumnFamilyDefinition[]{sampleTable};
+ }
+
+ @Override
+ public File getDBLocation(ConfigurationSource conf) {
+ return null;
+ }
+ };
+
+ try (DBStore dbStore = DBStoreBuilder.newBuilder(conf, sampleDB)
+ .setName("SampleStore").setPath(newFolder.toPath()).build()) {
+ Assert.assertTrue(dbStore instanceof RDBStore);
+ RDBStore rdbStore = (RDBStore) dbStore;
+ List<ColumnFamilyHandle> cfHandles = rdbStore.getColumnFamilyHandles();
+
+ // we also have the default column family, so there are 2
+ Assert.assertEquals(2, cfHandles.size());
+
+ boolean checked = false;
+ for (ColumnFamilyHandle cfHandle : cfHandles) {
+ if (Arrays.equals(cfHandle.getName(),
+ sampleTableName.getBytes(StandardCharsets.UTF_8))) {
+ // get the default value
+ boolean defaultValue = new ColumnFamilyOptions()
+ .forceConsistencyChecks();
+
+ // the value should be different from the default value
+ Assert.assertNotEquals(cfHandle.getDescriptor()
+ .getOptions().forceConsistencyChecks(), defaultValue);
+ checked = true;
+ }
+ }
+ Assert.assertTrue(checked);
+ }
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org