You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bh...@apache.org on 2018/12/06 19:40:30 UTC
[2/2] hadoop git commit: HDDS-864. Use strongly typed codec
implementations for the tables of the OmMetadataManager. Contributed by Elek
Marton.
HDDS-864. Use strongly typed codec implementations for the tables of the OmMetadataManager. Contributed by Elek Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/343aaea2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/343aaea2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/343aaea2
Branch: refs/heads/trunk
Commit: 343aaea2d12da0154273ff5f6eedc1ea5fae84cb
Parents: c03024a
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Thu Dec 6 11:39:59 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Thu Dec 6 11:39:59 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/utils/db/Codec.java | 3 +
.../apache/hadoop/utils/db/CodecRegistry.java | 19 ++
.../org/apache/hadoop/utils/db/DBStore.java | 11 +-
.../apache/hadoop/utils/db/DBStoreBuilder.java | 9 +-
.../org/apache/hadoop/utils/db/RDBStore.java | 106 ++++-------
.../org/apache/hadoop/utils/db/TypedTable.java | 3 +
.../hadoop/ozone/om/OMMetadataManager.java | 36 ++--
.../ozone/om/codec/OmBucketInfoCodec.java | 51 +++++
.../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 51 +++++
.../ozone/om/codec/OmVolumeArgsCodec.java | 51 +++++
.../hadoop/ozone/om/codec/VolumeListCodec.java | 50 +++++
.../hadoop/ozone/om/codec/package-info.java | 24 +++
.../hadoop/ozone/om/helpers/OmKeyInfo.java | 24 ++-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 3 +-
.../hadoop/ozone/om/TestOzoneManager.java | 8 +-
.../hadoop/ozone/om/BucketManagerImpl.java | 37 ++--
.../hadoop/ozone/om/KeyDeletingService.java | 3 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 88 ++++-----
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 188 +++++++++----------
.../hadoop/ozone/om/VolumeManagerImpl.java | 109 ++++-------
.../hadoop/ozone/om/TestBucketManagerImpl.java | 50 +++--
.../hadoop/ozone/om/TestKeyDeletingService.java | 41 ++--
.../hadoop/ozone/om/TestKeyManagerImpl.java | 72 +++++--
23 files changed, 638 insertions(+), 399 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
index 7f6f489..69637dc 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/Codec.java
@@ -28,11 +28,14 @@ public interface Codec<T> {
/**
* Convert object to raw persisted format.
+ * @param object The original java object. Should not be null.
*/
byte[] toPersistedFormat(T object);
/**
* Convert object from raw persisted format.
+ *
+ * @param rawData Byte array from the key/value store. Should not be null.
*/
T fromPersistedFormat(byte[] rawData);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
index 9bdff19..9c367ba 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/CodecRegistry.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.utils.db;
import java.util.HashMap;
import java.util.Map;
+import com.google.common.base.Preconditions;
+
/**
* Collection of available codecs.
*/
@@ -42,6 +44,9 @@ public class CodecRegistry {
* @return the object with the parsed field data
*/
public <T> T asObject(byte[] rawData, Class<T> format) {
+ if (rawData == null) {
+ return null;
+ }
if (valueCodecs.containsKey(format)) {
return (T) valueCodecs.get(format).fromPersistedFormat(rawData);
} else {
@@ -58,6 +63,8 @@ public class CodecRegistry {
* @return byte array to store it ini the kv store.
*/
public <T> byte[] asRawData(T object) {
+ Preconditions.checkNotNull(object,
+ "Null value shouldn't be persisted in the database");
Class<T> format = (Class<T>) object.getClass();
if (valueCodecs.containsKey(format)) {
Codec<T> codec = (Codec<T>) valueCodecs.get(format);
@@ -67,4 +74,16 @@ public class CodecRegistry {
"Codec is not registered for type: " + format);
}
}
+
+ /**
+ * Addds codec to the internal collection.
+ *
+ * @param type Type of the codec source/destination object.
+ * @param codec The codec itself.
+ * @param <T> The type of the codec
+ */
+ public <T> void addCodec(Class<T> type, Codec<T> codec) {
+ valueCodecs.put(type, codec);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
index 51b716b..23498bd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
@@ -77,7 +77,8 @@ public interface DBStore extends AutoCloseable {
* @param dest - Destination Table.
* @throws IOException on Failure
*/
- void move(byte[] key, Table source, Table dest) throws IOException;
+ <KEY, VALUE> void move(KEY key, Table<KEY, VALUE> source,
+ Table<KEY, VALUE> dest) throws IOException;
/**
* Moves a key from the Source Table to the destination Table and updates the
@@ -89,7 +90,8 @@ public interface DBStore extends AutoCloseable {
* @param dest - Destination Table.
* @throws IOException on Failure
*/
- void move(byte[] key, byte[] value, Table source, Table dest)
+ <KEY, VALUE> void move(KEY key, VALUE value, Table<KEY, VALUE> source,
+ Table<KEY, VALUE> dest)
throws IOException;
/**
@@ -105,8 +107,9 @@ public interface DBStore extends AutoCloseable {
* @param dest - Destination Table.
* @throws IOException on Failure
*/
- void move(byte[] sourceKey, byte[] destKey, byte[] value,
- Table source, Table dest) throws IOException;
+ <KEY, VALUE> void move(KEY sourceKey, KEY destKey, VALUE value,
+ Table<KEY, VALUE> source, Table<KEY, VALUE> dest)
+ throws IOException;
/**
* Returns an estimated count of keys in this DB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java
index 5ba9b9c..3459b20 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStoreBuilder.java
@@ -56,11 +56,13 @@ public final class DBStoreBuilder {
private Path dbPath;
private List<String> tableNames;
private Configuration configuration;
+ private CodecRegistry registry;
private DBStoreBuilder(Configuration configuration) {
tables = new HashSet<>();
tableNames = new LinkedList<>();
this.configuration = configuration;
+ this.registry = new CodecRegistry();
}
public static DBStoreBuilder newBuilder(Configuration configuration) {
@@ -82,6 +84,11 @@ public final class DBStoreBuilder {
return this;
}
+ public <T> DBStoreBuilder addCodec(Class<T> type, Codec<T> codec) {
+ registry.addCodec(type, codec);
+ return this;
+ }
+
public DBStoreBuilder addTable(String tableName, ColumnFamilyOptions option)
throws IOException {
TableConfig tableConfig = new TableConfig(tableName, option);
@@ -124,7 +131,7 @@ public final class DBStoreBuilder {
if (!dbFile.getParentFile().exists()) {
throw new IOException("The DB destination directory should exist.");
}
- return new RDBStore(dbFile, options, tables);
+ return new RDBStore(dbFile, options, tables, registry);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
index 8ed60ad..b79e81b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
@@ -19,32 +19,32 @@
package org.apache.hadoop.utils.db;
-import com.google.common.base.Preconditions;
+import javax.management.ObjectName;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.utils.RocksDBStoreMBean;
+
+import com.google.common.base.Preconditions;
import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.DBOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
-import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Hashtable;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
/**
* RocksDB Store that supports creating Tables in DB.
*/
@@ -59,13 +59,20 @@ public class RDBStore implements DBStore {
private final Hashtable<String, ColumnFamilyHandle> handleTable;
private ObjectName statMBeanName;
- public RDBStore(File dbFile, DBOptions options, Set<TableConfig> families)
+ @VisibleForTesting
+ public RDBStore(File dbFile, DBOptions options,
+ Set<TableConfig> families) throws IOException {
+ this(dbFile, options, families, new CodecRegistry());
+ }
+
+ public RDBStore(File dbFile, DBOptions options, Set<TableConfig> families,
+ CodecRegistry registry)
throws IOException {
Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
Preconditions.checkNotNull(families);
Preconditions.checkArgument(families.size() > 0);
handleTable = new Hashtable<>();
- codecRegistry = new CodecRegistry();
+ codecRegistry = registry;
final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
new ArrayList<>();
final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
@@ -161,70 +168,31 @@ public class RDBStore implements DBStore {
}
@Override
- public void move(byte[] key, Table source, Table dest) throws IOException {
- RDBTable sourceTable;
- RDBTable destTable;
- if (source instanceof RDBTable) {
- sourceTable = (RDBTable) source;
- } else {
- LOG.error("Unexpected Table type. Expected RocksTable Store for Source.");
- throw new IOException("Unexpected TableStore Type in source. Expected "
- + "RocksDBTable.");
- }
+ public <KEY, VALUE> void move(KEY key, Table<KEY, VALUE> source,
+ Table<KEY, VALUE> dest) throws IOException {
+ try (BatchOperation batchOperation = initBatchOperation()) {
- if (dest instanceof RDBTable) {
- destTable = (RDBTable) dest;
- } else {
- LOG.error("Unexpected Table type. Expected RocksTable Store for Dest.");
- throw new IOException("Unexpected TableStore Type in dest. Expected "
- + "RocksDBTable.");
- }
- try (WriteBatch batch = new WriteBatch()) {
- byte[] value = sourceTable.get(key);
- batch.put(destTable.getHandle(), key, value);
- batch.delete(sourceTable.getHandle(), key);
- db.write(writeOptions, batch);
- } catch (RocksDBException rockdbException) {
- LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(key));
- throw toIOException("Unable to move key: " + DFSUtil.bytes2String(key),
- rockdbException);
+ VALUE value = source.get(key);
+ dest.putWithBatch(batchOperation, key, value);
+ source.deleteWithBatch(batchOperation, key);
+ commitBatchOperation(batchOperation);
}
}
@Override
- public void move(byte[] key, byte[] value, Table source,
- Table dest) throws IOException {
+ public <KEY, VALUE> void move(KEY key, VALUE value, Table<KEY, VALUE> source,
+ Table<KEY, VALUE> dest) throws IOException {
move(key, key, value, source, dest);
}
@Override
- public void move(byte[] sourceKey, byte[] destKey, byte[] value, Table source,
- Table dest) throws IOException {
- RDBTable sourceTable;
- RDBTable destTable;
- if (source instanceof RDBTable) {
- sourceTable = (RDBTable) source;
- } else {
- LOG.error("Unexpected Table type. Expected RocksTable Store for Source.");
- throw new IOException("Unexpected TableStore Type in source. Expected "
- + "RocksDBTable.");
- }
-
- if (dest instanceof RDBTable) {
- destTable = (RDBTable) dest;
- } else {
- LOG.error("Unexpected Table type. Expected RocksTable Store for Dest.");
- throw new IOException("Unexpected TableStore Type in dest. Expected "
- + "RocksDBTable.");
- }
- try (WriteBatch batch = new WriteBatch()) {
- batch.put(destTable.getHandle(), destKey, value);
- batch.delete(sourceTable.getHandle(), sourceKey);
- db.write(writeOptions, batch);
- } catch (RocksDBException rockdbException) {
- LOG.error("Move of key failed. Key:{}", DFSUtil.bytes2String(sourceKey));
- throw toIOException("Unable to move key: " +
- DFSUtil.bytes2String(sourceKey), rockdbException);
+ public <KEY, VALUE> void move(KEY sourceKey, KEY destKey, VALUE value,
+ Table<KEY, VALUE> source,
+ Table<KEY, VALUE> dest) throws IOException {
+ try (BatchOperation batchOperation = initBatchOperation()) {
+ dest.putWithBatch(batchOperation, destKey, value);
+ source.deleteWithBatch(batchOperation, sourceKey);
+ commitBatchOperation(batchOperation);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
index 087752c..d0a33fd 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/TypedTable.java
@@ -162,6 +162,9 @@ public class TypedTable<KEY, VALUE> implements Table<KEY, VALUE> {
public TypedKeyValue seek(KEY key) {
byte[] keyBytes = codecRegistry.asRawData(key);
KeyValue<byte[], byte[]> result = rawIterator.seek(keyBytes);
+ if (result == null) {
+ return null;
+ }
return new TypedKeyValue(result);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 8989a68..0374056 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -16,17 +16,19 @@
*/
package org.apache.hadoop.ozone.om;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.Table;
-import java.io.IOException;
-import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
/**
* OM metadata manager interface.
@@ -65,14 +67,14 @@ public interface OMMetadataManager {
*
* @param volume - Volume name
*/
- byte[] getVolumeKey(String volume);
+ String getVolumeKey(String volume);
/**
* Given a user return the corresponding DB key.
*
* @param user - User name
*/
- byte[] getUserKey(String user);
+ String getUserKey(String user);
/**
* Given a volume and bucket, return the corresponding DB key.
@@ -80,17 +82,19 @@ public interface OMMetadataManager {
* @param volume - User name
* @param bucket - Bucket name
*/
- byte[] getBucketKey(String volume, String bucket);
+ String getBucketKey(String volume, String bucket);
/**
* Given a volume, bucket and a key, return the corresponding DB key.
*
* @param volume - volume name
* @param bucket - bucket name
- * @param key - key name
- * @return bytes of DB key.
+ * @param key - key name
+ * @return DB key as String.
*/
- byte[] getOzoneKeyBytes(String volume, String bucket, String key);
+
+ String getOzoneKey(String volume, String bucket, String key);
+
/**
* Returns the DB key name of a open key in OM metadata store. Should be
@@ -102,7 +106,7 @@ public interface OMMetadataManager {
* @param id - the id for this open
* @return bytes of DB key.
*/
- byte[] getOpenKeyBytes(String volume, String bucket, String key, long id);
+ String getOpenKey(String volume, String bucket, String key, long id);
/**
* Given a volume, check if it is empty, i.e there are no buckets inside it.
@@ -203,42 +207,42 @@ public interface OMMetadataManager {
*
* @return UserTable.
*/
- Table<byte[], byte[]> getUserTable();
+ Table<String, VolumeList> getUserTable();
/**
* Returns the Volume Table.
*
* @return VolumeTable.
*/
- Table<byte[], byte[]> getVolumeTable();
+ Table<String, OmVolumeArgs> getVolumeTable();
/**
* Returns the BucketTable.
*
* @return BucketTable.
*/
- Table<byte[], byte[]> getBucketTable();
+ Table<String, OmBucketInfo> getBucketTable();
/**
* Returns the KeyTable.
*
* @return KeyTable.
*/
- Table<byte[], byte[]> getKeyTable();
+ Table<String, OmKeyInfo> getKeyTable();
/**
* Get Deleted Table.
*
* @return Deleted Table.
*/
- Table<byte[], byte[]> getDeletedTable();
+ Table<String, OmKeyInfo> getDeletedTable();
/**
* Gets the OpenKeyTable.
*
* @return Table.
*/
- Table<byte[], byte[]> getOpenKeyTable();
+ Table<String, OmKeyInfo> getOpenKeyTable();
/**
* Gets the S3Bucket to Ozone Volume/bucket mapping table.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
new file mode 100644
index 0000000..7f5e5d0
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmBucketInfoCodec.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
+import org.apache.hadoop.utils.db.Codec;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Codec to encode OmBucketInfo as byte array.
+ */
+public class OmBucketInfoCodec implements Codec<OmBucketInfo> {
+
+ @Override
+ public byte[] toPersistedFormat(OmBucketInfo object) {
+ Preconditions
+ .checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.getProtobuf().toByteArray();
+ }
+
+ @Override
+ public OmBucketInfo fromPersistedFormat(byte[] rawData) {
+ Preconditions
+ .checkNotNull("Null byte array can't converted to real object.");
+ try {
+ return OmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(rawData));
+ } catch (InvalidProtocolBufferException e) {
+ throw new IllegalArgumentException(
+ "Can't encode the the raw data from the byte array", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
new file mode 100644
index 0000000..93aaeb4
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmKeyInfoCodec.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
+import org.apache.hadoop.utils.db.Codec;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Codec to encode OmKeyInfo as byte array.
+ */
+public class OmKeyInfoCodec implements Codec<OmKeyInfo> {
+
+ @Override
+ public byte[] toPersistedFormat(OmKeyInfo object) {
+ Preconditions
+ .checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.getProtobuf().toByteArray();
+ }
+
+ @Override
+ public OmKeyInfo fromPersistedFormat(byte[] rawData) {
+ Preconditions
+ .checkNotNull("Null byte array can't converted to real object.");
+ try {
+ return OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(rawData));
+ } catch (InvalidProtocolBufferException e) {
+ throw new IllegalArgumentException(
+ "Can't encode the the raw data from the byte array", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
new file mode 100644
index 0000000..f31d450
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/OmVolumeArgsCodec.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
+import org.apache.hadoop.utils.db.Codec;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Codec to encode OmVolumeArgsCodec as byte array.
+ */
+public class OmVolumeArgsCodec implements Codec<OmVolumeArgs> {
+
+ @Override
+ public byte[] toPersistedFormat(OmVolumeArgs object) {
+ Preconditions
+ .checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.getProtobuf().toByteArray();
+ }
+
+ @Override
+ public OmVolumeArgs fromPersistedFormat(byte[] rawData) {
+ Preconditions
+ .checkNotNull("Null byte array can't converted to real object.");
+ try {
+ return OmVolumeArgs.getFromProtobuf(VolumeInfo.parseFrom(rawData));
+ } catch (InvalidProtocolBufferException e) {
+ throw new IllegalArgumentException(
+ "Can't encode the the raw data from the byte array", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/VolumeListCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/VolumeListCodec.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/VolumeListCodec.java
new file mode 100644
index 0000000..f71da25
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/VolumeListCodec.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.codec;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
+import org.apache.hadoop.utils.db.Codec;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Codec to encode VolumeList as byte array.
+ */
+public class VolumeListCodec implements Codec<VolumeList> {
+
+ @Override
+ public byte[] toPersistedFormat(VolumeList object) {
+ Preconditions
+ .checkNotNull(object, "Null object can't be converted to byte array.");
+ return object.toByteArray();
+ }
+
+ @Override
+ public VolumeList fromPersistedFormat(byte[] rawData) {
+ Preconditions
+ .checkNotNull("Null byte array can't converted to real object.");
+ try {
+ return VolumeList.parseFrom(rawData);
+ } catch (InvalidProtocolBufferException e) {
+ throw new IllegalArgumentException(
+ "Can't encode the the raw data from the byte array", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
new file mode 100644
index 0000000..df6ed9c
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/codec/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * Utility classes to encode/decode DTO objects to/from byte array.
+ */
+
+/**
+ * Utility classes to encode/decode DTO objects to/from byte array.
+ */
+package org.apache.hadoop.ozone.om.codec;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index c9a7e48..9f2eb87 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -17,14 +17,16 @@
*/
package org.apache.hadoop.ozone.om.helpers;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.util.Time;
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
+import com.google.common.base.Preconditions;
/**
* Args for key block. The block instance for the key requested in putKey.
@@ -45,7 +47,8 @@ public final class OmKeyInfo {
private OmKeyInfo(String volumeName, String bucketName, String keyName,
List<OmKeyLocationInfoGroup> versions, long dataSize,
- long creationTime, long modificationTime, HddsProtos.ReplicationType type,
+ long creationTime, long modificationTime,
+ HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor) {
this.volumeName = volumeName;
this.bucketName = bucketName;
@@ -206,7 +209,8 @@ public final class OmKeyInfo {
private String bucketName;
private String keyName;
private long dataSize;
- private List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups;
+ private List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups =
+ new ArrayList<>();
private long creationTime;
private long modificationTime;
private HddsProtos.ReplicationType type;
@@ -248,13 +252,13 @@ public final class OmKeyInfo {
return this;
}
- public Builder setReplicationFactor(HddsProtos.ReplicationFactor factor) {
- this.factor = factor;
+ public Builder setReplicationFactor(HddsProtos.ReplicationFactor replFact) {
+ this.factor = replFact;
return this;
}
- public Builder setReplicationType(HddsProtos.ReplicationType type) {
- this.type = type;
+ public Builder setReplicationType(HddsProtos.ReplicationType replType) {
+ this.type = replType;
return this;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
index 165d9ab..4bbecaa 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
@@ -219,7 +219,8 @@ public final class OmVolumeArgs implements Auditable{
.setQuotaInBytes(quotaInBytes)
.addAllMetadata(metadataList)
.addAllVolumeAcls(aclList)
- .setCreationTime(creationTime)
+ .setCreationTime(
+ creationTime == 0 ? System.currentTimeMillis() : creationTime)
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
index 325ad2e..6f699c9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
@@ -32,10 +32,12 @@ import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
@@ -293,8 +295,8 @@ public class TestOzoneManager {
OMMetadataManager metadataManager =
cluster.getOzoneManager().getMetadataManager();
- byte[] userKey = metadataManager.getUserKey(userName);
- byte[] volumes = metadataManager.getUserTable().get(userKey);
+ String userKey = metadataManager.getUserKey(userName);
+ VolumeList volumes = metadataManager.getUserTable().get(userKey);
//that was the last volume of the user, shouldn't be any record here
Assert.assertNull(volumes);
@@ -653,7 +655,7 @@ public class TestOzoneManager {
// Make sure the deleted key has been moved to the deleted table.
OMMetadataManager manager = cluster.getOzoneManager().
getMetadataManager();
- try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> iter =
+ try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> iter =
manager.getDeletedTable().iterator()) {
iter.seekToFirst();
Table.KeyValue kv = iter.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
index b99b98e..63438ee 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
@@ -16,21 +16,21 @@
*/
package org.apache.hadoop.ozone.om;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.List;
+
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.util.Time;
+
+import com.google.common.base.Preconditions;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.List;
-
/**
* OM bucket manager.
*/
@@ -84,8 +84,8 @@ public class BucketManagerImpl implements BucketManager {
metadataManager.getLock().acquireVolumeLock(volumeName);
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
- byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ String volumeKey = metadataManager.getVolumeKey(volumeName);
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
//Check if the volume exists
if (metadataManager.getVolumeTable().get(volumeKey) == null) {
@@ -109,7 +109,7 @@ public class BucketManagerImpl implements BucketManager {
.setCreationTime(Time.now())
.build();
metadataManager.getBucketTable().put(bucketKey,
- omBucketInfo.getProtobuf().toByteArray());
+ omBucketInfo);
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
} catch (IOException | DBException ex) {
@@ -137,15 +137,15 @@ public class BucketManagerImpl implements BucketManager {
Preconditions.checkNotNull(bucketName);
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- byte[] value = metadataManager.getBucketTable().get(bucketKey);
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo value = metadataManager.getBucketTable().get(bucketKey);
if (value == null) {
LOG.debug("bucket: {} not found in volume: {}.", bucketName,
volumeName);
throw new OMException("Bucket not found",
OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
- return OmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(value));
+ return value;
} catch (IOException | DBException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Exception while getting bucket info for bucket: {}",
@@ -170,16 +170,15 @@ public class BucketManagerImpl implements BucketManager {
String bucketName = args.getBucketName();
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- byte[] value = metadataManager.getBucketTable().get(bucketKey);
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo oldBucketInfo =
+ metadataManager.getBucketTable().get(bucketKey);
//Check if bucket exist
- if (value == null) {
+ if (oldBucketInfo == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new OMException("Bucket doesn't exist",
OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
- OmBucketInfo oldBucketInfo = OmBucketInfo.getFromProtobuf(
- BucketInfo.parseFrom(value));
OmBucketInfo.Builder bucketInfoBuilder = OmBucketInfo.newBuilder();
bucketInfoBuilder.setVolumeName(oldBucketInfo.getVolumeName())
.setBucketName(oldBucketInfo.getBucketName());
@@ -216,8 +215,8 @@ public class BucketManagerImpl implements BucketManager {
}
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
- metadataManager.getBucketTable().put(bucketKey,
- bucketInfoBuilder.build().getProtobuf().toByteArray());
+ metadataManager.getBucketTable()
+ .put(bucketKey, bucketInfoBuilder.build());
} catch (IOException | DBException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
@@ -265,7 +264,7 @@ public class BucketManagerImpl implements BucketManager {
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
//Check if bucket exists
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
if (metadataManager.getBucketTable().get(bucketKey) == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new OMException("Bucket doesn't exist",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
index b7af87c..9c1dc2d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyDeletingService.java
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
import org.apache.hadoop.util.Time;
@@ -162,7 +161,7 @@ public class KeyDeletingService extends BackgroundService {
if (result.isSuccess()) {
// Purge key from OM DB.
deletedTable.deleteWithBatch(writeBatch,
- DFSUtil.string2Bytes(result.getObjectKey()));
+ result.getObjectKey());
LOG.debug("Key {} deleted from OM DB", result.getObjectKey());
deletedCount++;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 2c0b543..73daff6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -39,8 +38,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocationList;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BackgroundService;
import org.apache.hadoop.utils.db.BatchOperation;
@@ -125,8 +122,8 @@ public class KeyManagerImpl implements KeyManager {
private void validateBucket(String volumeName, String bucketName)
throws IOException {
- byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ String volumeKey = metadataManager.getVolumeKey(volumeName);
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
//Check if the volume exists
if (metadataManager.getVolumeTable().get(volumeKey) == null) {
@@ -150,18 +147,17 @@ public class KeyManagerImpl implements KeyManager {
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
validateBucket(volumeName, bucketName);
- byte[] openKey = metadataManager.getOpenKeyBytes(
+ String openKey = metadataManager.getOpenKey(
volumeName, bucketName, keyName, clientID);
- byte[] keyData = metadataManager.getOpenKeyTable().get(openKey);
- if (keyData == null) {
+ OmKeyInfo keyInfo = metadataManager.getOpenKeyTable().get(openKey);
+ if (keyInfo == null) {
LOG.error("Allocate block for a key not in open status in meta store" +
" /{}/{}/{} with ID {}", volumeName, bucketName, keyName, clientID);
throw new OMException("Open Key not found",
OMException.ResultCodes.FAILED_KEY_NOT_FOUND);
}
- OmKeyInfo keyInfo =
- OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
+
AllocatedBlock allocatedBlock;
try {
allocatedBlock =
@@ -184,7 +180,7 @@ public class KeyManagerImpl implements KeyManager {
keyInfo.appendNewBlocks(Collections.singletonList(info));
keyInfo.updateModifcationTime();
metadataManager.getOpenKeyTable().put(openKey,
- keyInfo.getProtobuf().toByteArray());
+ keyInfo);
return info;
}
@@ -214,7 +210,7 @@ public class KeyManagerImpl implements KeyManager {
try {
long requestedSize = Math.min(preallocateMax, args.getDataSize());
List<OmKeyLocationInfo> locations = new ArrayList<>();
- byte[] objectKey = metadataManager.getOzoneKeyBytes(
+ String objectKey = metadataManager.getOzoneKey(
volumeName, bucketName, keyName);
// requested size is not required but more like a optimization:
// SCM looks at the requested, if it 0, no block will be allocated at
@@ -250,12 +246,10 @@ public class KeyManagerImpl implements KeyManager {
// value, then this value is used, otherwise, we allocate a single block
// which is the current size, if read by the client.
long size = args.getDataSize() >= 0 ? args.getDataSize() : scmBlockSize;
- byte[] value = metadataManager.getKeyTable().get(objectKey);
- OmKeyInfo keyInfo;
+ OmKeyInfo keyInfo = metadataManager.getKeyTable().get(objectKey);
long openVersion;
- if (value != null) {
+ if (keyInfo != null) {
// the key already exist, the new blocks will be added as new version
- keyInfo = OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
// when locations.size = 0, the new version will have identical blocks
// as its previous version
openVersion = keyInfo.addNewVersion(locations);
@@ -278,7 +272,7 @@ public class KeyManagerImpl implements KeyManager {
.build();
openVersion = 0;
}
- byte[] openKey = metadataManager.getOpenKeyBytes(
+ String openKey = metadataManager.getOpenKey(
volumeName, bucketName, keyName, currentTime);
if (metadataManager.getOpenKeyTable().get(openKey) != null) {
// This should not happen. If this condition is satisfied, it means
@@ -293,8 +287,7 @@ public class KeyManagerImpl implements KeyManager {
throw new OMException("Cannot allocate key. Not able to get a valid" +
"open key id.", OMException.ResultCodes.FAILED_KEY_ALLOCATION);
}
- metadataManager.getOpenKeyTable().put(openKey,
- keyInfo.getProtobuf().toByteArray());
+ metadataManager.getOpenKeyTable().put(openKey, keyInfo);
LOG.debug("Key {} allocated in volume {} bucket {}",
keyName, volumeName, bucketName);
return new OpenKeySession(currentTime, keyInfo, openVersion);
@@ -319,17 +312,15 @@ public class KeyManagerImpl implements KeyManager {
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
validateBucket(volumeName, bucketName);
- byte[] openKey = metadataManager.getOpenKeyBytes(volumeName, bucketName,
+ String openKey = metadataManager.getOpenKey(volumeName, bucketName,
keyName, clientID);
- byte[] objectKey = metadataManager.getOzoneKeyBytes(
+ String objectKey = metadataManager.getOzoneKey(
volumeName, bucketName, keyName);
- byte[] openKeyData = metadataManager.getOpenKeyTable().get(openKey);
- if (openKeyData == null) {
+ OmKeyInfo keyInfo = metadataManager.getOpenKeyTable().get(openKey);
+ if (keyInfo == null) {
throw new OMException("Commit a key without corresponding entry " +
- DFSUtil.bytes2String(objectKey), ResultCodes.FAILED_KEY_NOT_FOUND);
+ objectKey, ResultCodes.FAILED_KEY_NOT_FOUND);
}
- OmKeyInfo keyInfo =
- OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
keyInfo.setDataSize(args.getDataSize());
keyInfo.setModificationTime(Time.now());
List<OmKeyLocationInfo> locationInfoList = args.getLocationInfoList();
@@ -337,8 +328,10 @@ public class KeyManagerImpl implements KeyManager {
//update the block length for each block
keyInfo.updateLocationInfoList(locationInfoList);
- metadataManager.getStore().move(openKey, objectKey,
- keyInfo.getProtobuf().toByteArray(),
+ metadataManager.getStore().move(
+ openKey,
+ objectKey,
+ keyInfo,
metadataManager.getOpenKeyTable(),
metadataManager.getKeyTable());
} catch (OMException e) {
@@ -361,16 +354,16 @@ public class KeyManagerImpl implements KeyManager {
String keyName = args.getKeyName();
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
- byte[] keyBytes = metadataManager.getOzoneKeyBytes(
+ String keyBytes = metadataManager.getOzoneKey(
volumeName, bucketName, keyName);
- byte[] value = metadataManager.getKeyTable().get(keyBytes);
+ OmKeyInfo value = metadataManager.getKeyTable().get(keyBytes);
if (value == null) {
LOG.debug("volume:{} bucket:{} Key:{} not found",
volumeName, bucketName, keyName);
throw new OMException("Key not found",
OMException.ResultCodes.FAILED_KEY_NOT_FOUND);
}
- return OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
+ return value;
} catch (IOException ex) {
LOG.debug("Get key failed for volume:{} bucket:{} key:{}",
volumeName, bucketName, keyName, ex);
@@ -398,9 +391,9 @@ public class KeyManagerImpl implements KeyManager {
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
// fromKeyName should exist
- byte[] fromKey = metadataManager.getOzoneKeyBytes(
+ String fromKey = metadataManager.getOzoneKey(
volumeName, bucketName, fromKeyName);
- byte[] fromKeyValue = metadataManager.getKeyTable().get(fromKey);
+ OmKeyInfo fromKeyValue = metadataManager.getKeyTable().get(fromKey);
if (fromKeyValue == null) {
// TODO: Add support for renaming open key
LOG.error(
@@ -418,9 +411,9 @@ public class KeyManagerImpl implements KeyManager {
}
// toKeyName should not exist
- byte[] toKey =
- metadataManager.getOzoneKeyBytes(volumeName, bucketName, toKeyName);
- byte[] toKeyValue = metadataManager.getKeyTable().get(toKey);
+ String toKey =
+ metadataManager.getOzoneKey(volumeName, bucketName, toKeyName);
+ OmKeyInfo toKeyValue = metadataManager.getKeyTable().get(toKey);
if (toKeyValue != null) {
LOG.error(
"Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}. "
@@ -430,16 +423,13 @@ public class KeyManagerImpl implements KeyManager {
OMException.ResultCodes.FAILED_KEY_ALREADY_EXISTS);
}
-
- OmKeyInfo newKeyInfo =
- OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(fromKeyValue));
- newKeyInfo.setKeyName(toKeyName);
- newKeyInfo.updateModifcationTime();
+ fromKeyValue.setKeyName(toKeyName);
+ fromKeyValue.updateModifcationTime();
DBStore store = metadataManager.getStore();
try (BatchOperation batch = store.initBatchOperation()) {
metadataManager.getKeyTable().deleteWithBatch(batch, fromKey);
metadataManager.getKeyTable().putWithBatch(batch, toKey,
- newKeyInfo.getProtobuf().toByteArray());
+ fromKeyValue);
store.commitBatchOperation(batch);
}
} catch (IOException ex) {
@@ -460,16 +450,15 @@ public class KeyManagerImpl implements KeyManager {
String keyName = args.getKeyName();
metadataManager.getLock().acquireBucketLock(volumeName, bucketName);
try {
- byte[] objectKey = metadataManager.getOzoneKeyBytes(
+ String objectKey = metadataManager.getOzoneKey(
volumeName, bucketName, keyName);
- byte[] objectValue = metadataManager.getKeyTable().get(objectKey);
- if (objectValue == null) {
+ OmKeyInfo keyInfo = metadataManager.getKeyTable().get(objectKey);
+ if (keyInfo == null) {
throw new OMException("Key not found",
OMException.ResultCodes.FAILED_KEY_NOT_FOUND);
} else {
// directly delete key with no blocks from db. This key need not be
// moved to deleted table.
- KeyInfo keyInfo = KeyInfo.parseFrom(objectValue);
if (isKeyEmpty(keyInfo)) {
metadataManager.getKeyTable().delete(objectKey);
LOG.debug("Key {} deleted from OM DB", keyName);
@@ -491,9 +480,10 @@ public class KeyManagerImpl implements KeyManager {
}
}
- private boolean isKeyEmpty(KeyInfo keyInfo) {
- for (KeyLocationList keyLocationList : keyInfo.getKeyLocationListList()) {
- if (keyLocationList.getKeyLocationsCount() != 0) {
+ private boolean isKeyEmpty(OmKeyInfo keyInfo) {
+ for (OmKeyLocationInfoGroup keyLocationList : keyInfo
+ .getKeyLocationVersions()) {
+ if (keyLocationList.getLocationList().size() != 0) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 706a10f..cfd4a20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -20,27 +20,24 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.codec.OmBucketInfoCodec;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.codec.OmKeyInfoCodec;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
+import org.apache.hadoop.ozone.om.codec.OmVolumeArgsCodec;
+import org.apache.hadoop.ozone.om.codec.VolumeListCodec;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.db.DBStore;
@@ -123,32 +120,32 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
}
@Override
- public Table<byte[], byte[]> getUserTable() {
+ public Table<String, VolumeList> getUserTable() {
return userTable;
}
@Override
- public Table<byte[], byte[]> getVolumeTable() {
+ public Table<String, OmVolumeArgs> getVolumeTable() {
return volumeTable;
}
@Override
- public Table<byte[], byte[]> getBucketTable() {
+ public Table<String, OmBucketInfo> getBucketTable() {
return bucketTable;
}
@Override
- public Table<byte[], byte[]> getKeyTable() {
+ public Table<String, OmKeyInfo> getKeyTable() {
return keyTable;
}
@Override
- public Table<byte[], byte[]> getDeletedTable() {
+ public Table<String, OmKeyInfo> getDeletedTable() {
return deletedTable;
}
@Override
- public Table<byte[], byte[]> getOpenKeyTable() {
+ public Table<String, OmKeyInfo> getOpenKeyTable() {
return openKeyTable;
}
@@ -178,6 +175,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
// db, so we need to create the store object and initialize DB.
if (store == null) {
File metaDir = OmUtils.getOmDbDir(configuration);
+
this.store = DBStoreBuilder.newBuilder(configuration)
.setName(OM_DB_NAME)
.setPath(Paths.get(metaDir.getPath()))
@@ -188,28 +186,39 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
.addTable(DELETED_TABLE)
.addTable(OPEN_KEY_TABLE)
.addTable(S3_TABLE)
+ .addCodec(OmKeyInfo.class, new OmKeyInfoCodec())
+ .addCodec(OmBucketInfo.class, new OmBucketInfoCodec())
+ .addCodec(OmVolumeArgs.class, new OmVolumeArgsCodec())
+ .addCodec(VolumeList.class, new VolumeListCodec())
.build();
- userTable = this.store.getTable(USER_TABLE);
+ userTable =
+ this.store.getTable(USER_TABLE, String.class, VolumeList.class);
checkTableStatus(userTable, USER_TABLE);
-
- volumeTable = this.store.getTable(VOLUME_TABLE);
+ this.store.getTable(VOLUME_TABLE, String.class,
+ String.class);
+ volumeTable =
+ this.store.getTable(VOLUME_TABLE, String.class, OmVolumeArgs.class);
checkTableStatus(volumeTable, VOLUME_TABLE);
- bucketTable = this.store.getTable(BUCKET_TABLE);
+ bucketTable =
+ this.store.getTable(BUCKET_TABLE, String.class, OmBucketInfo.class);
checkTableStatus(bucketTable, BUCKET_TABLE);
- keyTable = this.store.getTable(KEY_TABLE);
+ keyTable = this.store.getTable(KEY_TABLE, String.class, OmKeyInfo.class);
checkTableStatus(keyTable, KEY_TABLE);
- deletedTable = this.store.getTable(DELETED_TABLE);
+ deletedTable =
+ this.store.getTable(DELETED_TABLE, String.class, OmKeyInfo.class);
checkTableStatus(deletedTable, DELETED_TABLE);
- openKeyTable = this.store.getTable(OPEN_KEY_TABLE);
+ openKeyTable =
+ this.store.getTable(OPEN_KEY_TABLE, String.class, OmKeyInfo.class);
checkTableStatus(openKeyTable, OPEN_KEY_TABLE);
s3Table = this.store.getTable(S3_TABLE);
checkTableStatus(s3Table, S3_TABLE);
+
}
}
@@ -241,8 +250,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* @param volume - Volume name
*/
@Override
- public byte[] getVolumeKey(String volume) {
- return DFSUtil.string2Bytes(OzoneConsts.OM_KEY_PREFIX + volume);
+ public String getVolumeKey(String volume) {
+ return OzoneConsts.OM_KEY_PREFIX + volume;
}
/**
@@ -251,8 +260,8 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* @param user - User name
*/
@Override
- public byte[] getUserKey(String user) {
- return DFSUtil.string2Bytes(user);
+ public String getUserKey(String user) {
+ return user;
}
/**
@@ -262,18 +271,18 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* @param bucket - Bucket name
*/
@Override
- public byte[] getBucketKey(String volume, String bucket) {
+ public String getBucketKey(String volume, String bucket) {
StringBuilder builder =
new StringBuilder().append(OM_KEY_PREFIX).append(volume);
if (StringUtils.isNotBlank(bucket)) {
builder.append(OM_KEY_PREFIX).append(bucket);
}
- return DFSUtil.string2Bytes(builder.toString());
+ return builder.toString();
}
@Override
- public byte[] getOzoneKeyBytes(String volume, String bucket, String key) {
+ public String getOzoneKey(String volume, String bucket, String key) {
StringBuilder builder = new StringBuilder()
.append(OM_KEY_PREFIX).append(volume);
// TODO : Throw if the Bucket is null?
@@ -281,15 +290,15 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
if (StringUtil.isNotBlank(key)) {
builder.append(OM_KEY_PREFIX).append(key);
}
- return DFSUtil.string2Bytes(builder.toString());
+ return builder.toString();
}
@Override
- public byte[] getOpenKeyBytes(String volume, String bucket,
- String key, long id) {
+ public String getOpenKey(String volume, String bucket,
+ String key, long id) {
String openKey = OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket +
OM_KEY_PREFIX + key + OM_KEY_PREFIX + id;
- return DFSUtil.string2Bytes(openKey);
+ return openKey;
}
/**
@@ -349,11 +358,12 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
*/
@Override
public boolean isVolumeEmpty(String volume) throws IOException {
- byte[] volumePrefix = getVolumeKey(volume + OM_KEY_PREFIX);
- try (TableIterator<byte[], Table.KeyValue> bucketIter = bucketTable
- .iterator()) {
- Table.KeyValue<byte[], byte[]> kv = bucketIter.seek(volumePrefix);
- if (kv != null && startsWith(kv.getKey(), volumePrefix)) {
+ String volumePrefix = getVolumeKey(volume + OM_KEY_PREFIX);
+
+ try (TableIterator<String, ? extends KeyValue<String, OmBucketInfo>>
+ bucketIter = bucketTable.iterator()) {
+ KeyValue<String, OmBucketInfo> kv = bucketIter.seek(volumePrefix);
+ if (kv != null && kv.getKey().startsWith(volumePrefix)) {
return false; // we found at least one bucket with this volume prefix.
}
}
@@ -371,10 +381,11 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
@Override
public boolean isBucketEmpty(String volume, String bucket)
throws IOException {
- byte[] keyPrefix = getBucketKey(volume, bucket + OM_KEY_PREFIX);
- try (TableIterator<byte[], Table.KeyValue> keyIter = keyTable.iterator()) {
- Table.KeyValue<byte[], byte[]> kv = keyIter.seek(keyPrefix);
- if (kv != null && startsWith(kv.getKey(), keyPrefix)) {
+ String keyPrefix = getBucketKey(volume, bucket);
+ try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> keyIter =
+ keyTable.iterator()) {
+ KeyValue<String, OmKeyInfo> kv = keyIter.seek(keyPrefix);
+ if (kv != null && kv.getKey().startsWith(keyPrefix)) {
return false; // we found at least one key with this vol/bucket prefix.
}
}
@@ -394,14 +405,13 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- byte[] volumeNameBytes = getVolumeKey(volumeName);
+ String volumeNameBytes = getVolumeKey(volumeName);
if (volumeTable.get(volumeNameBytes) == null) {
throw new OMException("Volume " + volumeName + " not found.",
ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
-
- byte[] startKey;
+ String startKey;
boolean skipStartKey = false;
if (StringUtil.isNotBlank(startBucket)) {
// if the user has specified a start key, we need to seek to that key
@@ -417,26 +427,26 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
startKey = getBucketKey(volumeName, bucketPrefix);
}
- byte[] seekPrefix;
+ String seekPrefix;
if (StringUtil.isNotBlank(bucketPrefix)) {
seekPrefix = getBucketKey(volumeName, bucketPrefix);
} else {
seekPrefix = getVolumeKey(volumeName + OM_KEY_PREFIX);
}
int currentCount = 0;
- try (TableIterator<byte[], Table.KeyValue> bucketIter = bucketTable
- .iterator()) {
- Table.KeyValue<byte[], byte[]> kv = bucketIter.seek(startKey);
+
+ try (TableIterator<String, ? extends KeyValue<String, OmBucketInfo>>
+ bucketIter = bucketTable.iterator()) {
+ KeyValue<String, OmBucketInfo> kv = bucketIter.seek(startKey);
while (currentCount < maxNumOfBuckets && bucketIter.hasNext()) {
kv = bucketIter.next();
// Skip the Start Bucket if needed.
if (kv != null && skipStartKey &&
- Arrays.equals(kv.getKey(), startKey)) {
+ kv.getKey().equals(startKey)) {
continue;
}
- if (kv != null && startsWith(kv.getKey(), seekPrefix)) {
- result.add(OmBucketInfo.getFromProtobuf(
- BucketInfo.parseFrom(kv.getValue())));
+ if (kv != null && kv.getKey().startsWith(seekPrefix)) {
+ result.add(kv.getValue());
currentCount++;
} else {
// The SeekPrefix does not match any more, we can break out of the
@@ -462,43 +472,42 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
- byte[] bucketNameBytes = getBucketKey(volumeName, bucketName);
+ String bucketNameBytes = getBucketKey(volumeName, bucketName);
if (getBucketTable().get(bucketNameBytes) == null) {
throw new OMException("Bucket " + bucketName + " not found.",
ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
- byte[] seekKey;
+ String seekKey;
boolean skipStartKey = false;
if (StringUtil.isNotBlank(startKey)) {
// Seek to the specified key.
- seekKey = getOzoneKeyBytes(volumeName, bucketName, startKey);
+ seekKey = getOzoneKey(volumeName, bucketName, startKey);
skipStartKey = true;
} else {
// This allows us to seek directly to the first key with the right prefix.
- seekKey = getOzoneKeyBytes(volumeName, bucketName, keyPrefix);
+ seekKey = getOzoneKey(volumeName, bucketName, keyPrefix);
}
- byte[] seekPrefix;
+ String seekPrefix;
if (StringUtil.isNotBlank(keyPrefix)) {
- seekPrefix = getOzoneKeyBytes(volumeName, bucketName, keyPrefix);
+ seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix);
} else {
seekPrefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
}
int currentCount = 0;
- try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> keyIter =
+ try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> keyIter =
getKeyTable()
.iterator()) {
- Table.KeyValue<byte[], byte[]> kv = keyIter.seek(seekKey);
+ KeyValue<String, OmKeyInfo> kv = keyIter.seek(seekKey);
while (currentCount < maxKeys && keyIter.hasNext()) {
kv = keyIter.next();
// Skip the Start key if needed.
- if (kv != null && skipStartKey && Arrays.equals(kv.getKey(), seekKey)) {
+ if (kv != null && skipStartKey && kv.getKey().equals(seekKey)) {
continue;
}
- if (kv != null && startsWith(kv.getKey(), seekPrefix)) {
- result.add(OmKeyInfo.getFromProtobuf(
- KeyInfo.parseFrom(kv.getValue())));
+ if (kv != null && kv.getKey().startsWith(seekPrefix)) {
+ result.add(kv.getValue());
currentCount++;
} else {
// The SeekPrefix does not match any more, we can break out of the
@@ -538,8 +547,9 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
continue;
}
if (startKeyFound && result.size() < maxKeys) {
- byte[] volumeInfo = getVolumeTable().get(this.getVolumeKey(volumeName));
- if (volumeInfo == null) {
+ OmVolumeArgs volumeArgs =
+ getVolumeTable().get(this.getVolumeKey(volumeName));
+ if (volumeArgs == null) {
// Could not get volume info by given volume name,
// since the volume name is loaded from db,
// this probably means om db is corrupted or some entries are
@@ -547,8 +557,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
throw new OMException("Volume info not found for " + volumeName,
ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- VolumeInfo info = VolumeInfo.parseFrom(volumeInfo);
- OmVolumeArgs volumeArgs = OmVolumeArgs.getFromProtobuf(info);
result.add(volumeArgs);
}
}
@@ -556,49 +564,42 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
return result;
}
- private VolumeList getVolumesByUser(String userName)
+ private VolumeList getVolumesByUser(String userNameKey)
throws OMException {
- return getVolumesByUser(getUserKey(userName));
- }
-
- private VolumeList getVolumesByUser(byte[] userNameKey)
- throws OMException {
- VolumeList volumes = null;
try {
- byte[] volumesInBytes = getUserTable().get(userNameKey);
- if (volumesInBytes == null) {
+ VolumeList volumeList = getUserTable().get(userNameKey);
+ if (volumeList == null) {
// No volume found for this user, return an empty list
return VolumeList.newBuilder().build();
+ } else {
+ return volumeList;
}
- volumes = VolumeList.parseFrom(volumesInBytes);
} catch (IOException e) {
throw new OMException("Unable to get volumes info by the given user, "
+ "metadata might be corrupted", e,
ResultCodes.FAILED_METADATA_ERROR);
}
- return volumes;
}
@Override
public List<BlockGroup> getPendingDeletionKeys(final int keyCount)
throws IOException {
List<BlockGroup> keyBlocksList = Lists.newArrayList();
- try (TableIterator<byte[], ? extends KeyValue<byte[], byte[]>> keyIter =
+ try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>> keyIter =
getDeletedTable()
.iterator()) {
int currentCount = 0;
while (keyIter.hasNext() && currentCount < keyCount) {
- KeyValue<byte[], byte[]> kv = keyIter.next();
+ KeyValue<String, OmKeyInfo> kv = keyIter.next();
if (kv != null) {
- OmKeyInfo info =
- OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(kv.getValue()));
+ OmKeyInfo info = kv.getValue();
// Get block keys as a list.
OmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
List<BlockID> item = latest.getLocationList().stream()
.map(b -> new BlockID(b.getContainerID(), b.getLocalID()))
.collect(Collectors.toList());
BlockGroup keyBlocks = BlockGroup.newBuilder()
- .setKeyName(DFSUtil.bytes2String(kv.getKey()))
+ .setKeyName(kv.getKey())
.addAllBlockIDs(item)
.build();
keyBlocksList.add(keyBlocks);
@@ -614,27 +615,6 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
List<BlockGroup> keyBlocksList = Lists.newArrayList();
long now = Time.now();
// TODO: Fix the getExpiredOpenKeys, Not part of this patch.
- List<Map.Entry<byte[], byte[]>> rangeResult = Collections.emptyList();
-
- for (Map.Entry<byte[], byte[]> entry : rangeResult) {
- OmKeyInfo info =
- OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));
- long lastModify = info.getModificationTime();
- if (now - lastModify < this.openKeyExpireThresholdMS) {
- // consider as may still be active, not hanging.
- continue;
- }
- // Get block keys as a list.
- List<BlockID> item = info.getLatestVersionLocations()
- .getBlocksLatestVersionOnly().stream()
- .map(b -> new BlockID(b.getContainerID(), b.getLocalID()))
- .collect(Collectors.toList());
- BlockGroup keyBlocks = BlockGroup.newBuilder()
- .setKeyName(DFSUtil.bytes2String(entry.getKey()))
- .addAllBlockIDs(item)
- .build();
- keyBlocksList.add(keyBlocks);
- }
return keyBlocksList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/343aaea2/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
index 36d245b..9bf8de1 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/VolumeManagerImpl.java
@@ -19,16 +19,12 @@ package org.apache.hadoop.ozone.om;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.RocksDBStore;
import org.apache.hadoop.utils.db.BatchOperation;
@@ -66,12 +62,11 @@ public class VolumeManagerImpl implements VolumeManager {
private void addVolumeToOwnerList(String volume, String owner,
BatchOperation batchOperation) throws IOException {
// Get the volume list
- byte[] dbUserKey = metadataManager.getUserKey(owner);
- byte[] volumeList = metadataManager.getUserTable().get(dbUserKey);
+ String dbUserKey = metadataManager.getUserKey(owner);
+ VolumeList volumeList = metadataManager.getUserTable().get(dbUserKey);
List<String> prevVolList = new ArrayList<>();
if (volumeList != null) {
- VolumeList vlist = VolumeList.parseFrom(volumeList);
- prevVolList.addAll(vlist.getVolumeNamesList());
+ prevVolList.addAll(volumeList.getVolumeNamesList());
}
// Check the volume count
@@ -85,18 +80,17 @@ public class VolumeManagerImpl implements VolumeManager {
VolumeList newVolList = VolumeList.newBuilder()
.addAllVolumeNames(prevVolList).build();
metadataManager.getUserTable().putWithBatch(batchOperation,
- dbUserKey, newVolList.toByteArray());
+ dbUserKey, newVolList);
}
private void delVolumeFromOwnerList(String volume, String owner,
BatchOperation batch) throws RocksDBException, IOException {
// Get the volume list
- byte[] dbUserKey = metadataManager.getUserKey(owner);
- byte[] volumeList = metadataManager.getUserTable().get(dbUserKey);
+ String dbUserKey = metadataManager.getUserKey(owner);
+ VolumeList volumeList = metadataManager.getUserTable().get(dbUserKey);
List<String> prevVolList = new ArrayList<>();
if (volumeList != null) {
- VolumeList vlist = VolumeList.parseFrom(volumeList);
- prevVolList.addAll(vlist.getVolumeNamesList());
+ prevVolList.addAll(volumeList.getVolumeNamesList());
} else {
LOG.debug("volume:{} not found for user:{}");
throw new OMException(ResultCodes.FAILED_USER_NOT_FOUND);
@@ -110,7 +104,7 @@ public class VolumeManagerImpl implements VolumeManager {
VolumeList newVolList = VolumeList.newBuilder()
.addAllVolumeNames(prevVolList).build();
metadataManager.getUserTable().putWithBatch(batch,
- dbUserKey, newVolList.toByteArray());
+ dbUserKey, newVolList);
}
}
@@ -124,8 +118,9 @@ public class VolumeManagerImpl implements VolumeManager {
metadataManager.getLock().acquireUserLock(args.getOwnerName());
metadataManager.getLock().acquireVolumeLock(args.getVolume());
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(args.getVolume());
- byte[] volumeInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
+ String dbVolumeKey = metadataManager.getVolumeKey(args.getVolume());
+ OmVolumeArgs volumeInfo =
+ metadataManager.getVolumeTable().get(dbVolumeKey);
// Check of the volume already exists
if (volumeInfo != null) {
@@ -136,25 +131,8 @@ public class VolumeManagerImpl implements VolumeManager {
try (BatchOperation batch = metadataManager.getStore()
.initBatchOperation()) {
// Write the vol info
- List<HddsProtos.KeyValue> metadataList = new ArrayList<>();
- for (Map.Entry<String, String> entry :
- args.getKeyValueMap().entrySet()) {
- metadataList.add(HddsProtos.KeyValue.newBuilder()
- .setKey(entry.getKey()).setValue(entry.getValue()).build());
- }
- List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
-
- VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
- .setAdminName(args.getAdminName())
- .setOwnerName(args.getOwnerName())
- .setVolume(args.getVolume())
- .setQuotaInBytes(args.getQuotaInBytes())
- .addAllMetadata(metadataList)
- .addAllVolumeAcls(aclList)
- .setCreationTime(Time.now())
- .build();
metadataManager.getVolumeTable().putWithBatch(batch,
- dbVolumeKey, newVolumeInfo.toByteArray());
+ dbVolumeKey, args);
// Add volume to user list
addVolumeToOwnerList(args.getVolume(), args.getOwnerName(), batch);
@@ -189,17 +167,16 @@ public class VolumeManagerImpl implements VolumeManager {
metadataManager.getLock().acquireUserLock(owner);
metadataManager.getLock().acquireVolumeLock(volume);
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
- if (volInfo == null) {
+ String dbVolumeKey = metadataManager.getVolumeKey(volume);
+ OmVolumeArgs volumeArgs = metadataManager
+ .getVolumeTable().get(dbVolumeKey);
+ if (volumeArgs == null) {
LOG.debug("Changing volume ownership failed for user:{} volume:{}",
owner, volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- OmVolumeArgs volumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
+ Preconditions.checkState(volume.equals(volumeArgs.getVolume()));
try (BatchOperation batch = metadataManager.getStore()
.initBatchOperation()) {
@@ -214,9 +191,8 @@ public class VolumeManagerImpl implements VolumeManager {
.setCreationTime(volumeArgs.getCreationTime())
.build();
- VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
metadataManager.getVolumeTable().putWithBatch(batch,
- dbVolumeKey, newVolumeInfo.toByteArray());
+ dbVolumeKey, newVolumeArgs);
metadataManager.getStore().commitBatchOperation(batch);
}
} catch (RocksDBException | IOException ex) {
@@ -248,16 +224,15 @@ public class VolumeManagerImpl implements VolumeManager {
Preconditions.checkNotNull(volume);
metadataManager.getLock().acquireVolumeLock(volume);
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
- if (volInfo == null) {
+ String dbVolumeKey = metadataManager.getVolumeKey(volume);
+ OmVolumeArgs volumeArgs =
+ metadataManager.getVolumeTable().get(dbVolumeKey);
+ if (volumeArgs == null) {
LOG.debug("volume:{} does not exist", volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- OmVolumeArgs volumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
+ Preconditions.checkState(volume.equals(volumeArgs.getVolume()));
OmVolumeArgs newVolumeArgs =
OmVolumeArgs.newBuilder()
@@ -267,9 +242,7 @@ public class VolumeManagerImpl implements VolumeManager {
.setQuotaInBytes(quota)
.setCreationTime(volumeArgs.getCreationTime()).build();
- VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
- metadataManager.getVolumeTable().put(dbVolumeKey,
- newVolumeInfo.toByteArray());
+ metadataManager.getVolumeTable().put(dbVolumeKey, newVolumeArgs);
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Changing volume quota failed for volume:{} quota:{}", volume,
@@ -292,16 +265,14 @@ public class VolumeManagerImpl implements VolumeManager {
Preconditions.checkNotNull(volume);
metadataManager.getLock().acquireVolumeLock(volume);
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
- if (volInfo == null) {
+ String dbVolumeKey = metadataManager.getVolumeKey(volume);
+ OmVolumeArgs volumeArgs =
+ metadataManager.getVolumeTable().get(dbVolumeKey);
+ if (volumeArgs == null) {
LOG.debug("volume:{} does not exist", volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- OmVolumeArgs volumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
return volumeArgs;
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
@@ -333,9 +304,10 @@ public class VolumeManagerImpl implements VolumeManager {
metadataManager.getLock().acquireVolumeLock(volume);
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
- if (volInfo == null) {
+ String dbVolumeKey = metadataManager.getVolumeKey(volume);
+ OmVolumeArgs volumeArgs =
+ metadataManager.getVolumeTable().get(dbVolumeKey);
+ if (volumeArgs == null) {
LOG.debug("volume:{} does not exist", volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
@@ -344,14 +316,12 @@ public class VolumeManagerImpl implements VolumeManager {
LOG.debug("volume:{} is not empty", volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_EMPTY);
}
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
+ Preconditions.checkState(volume.equals(volumeArgs.getVolume()));
// delete the volume from the owner list
// as well as delete the volume entry
try (BatchOperation batch = metadataManager.getStore()
.initBatchOperation()) {
- delVolumeFromOwnerList(volume, volumeInfo.getOwnerName(), batch);
+ delVolumeFromOwnerList(volume, volumeArgs.getOwnerName(), batch);
metadataManager.getVolumeTable().deleteWithBatch(batch, dbVolumeKey);
metadataManager.getStore().commitBatchOperation(batch);
}
@@ -386,16 +356,15 @@ public class VolumeManagerImpl implements VolumeManager {
Preconditions.checkNotNull(userAcl);
metadataManager.getLock().acquireVolumeLock(volume);
try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.getVolumeTable().get(dbVolumeKey);
- if (volInfo == null) {
+ String dbVolumeKey = metadataManager.getVolumeKey(volume);
+ OmVolumeArgs volumeArgs =
+ metadataManager.getVolumeTable().get(dbVolumeKey);
+ if (volumeArgs == null) {
LOG.debug("volume:{} does not exist", volume);
throw new OMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- OmVolumeArgs volumeArgs = OmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
+ Preconditions.checkState(volume.equals(volumeArgs.getVolume()));
return volumeArgs.getAclMap().hasAccess(userAcl);
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org