You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/02/29 23:29:34 UTC
hadoop git commit: HDFS-9853. Ozone: Add container definitions.
Contributed by Anu Engineer.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 1b7df69d3 -> f3fbae8b8
HDFS-9853. Ozone: Add container definitions. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3fbae8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3fbae8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3fbae8b
Branch: refs/heads/HDFS-7240
Commit: f3fbae8b84d4a9dcd9713d379e46fd2348c745df
Parents: 1b7df69
Author: Anu Engineer <ae...@apache.org>
Authored: Mon Feb 29 14:26:49 2016 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Mon Feb 29 14:26:49 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
.../org/apache/hadoop/ozone/package-info.java | 35 +++
.../main/proto/DatanodeContainerProtocol.proto | 314 +++++++++++++++++++
3 files changed, 350 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index f1d5af9..4fc1588 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -351,6 +351,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
+ <include>DatanodeContainerProtocol.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
new file mode 100644
index 0000000..db399db
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+/**
+ This package contains class that support ozone implementation on the datanode
+ side.
+
+ Main parts of ozone on datanode are:
+
+ 1. REST Interface - This code lives under the web directory and listens to the
+ WebHDFS port.
+
+ 2. Datanode container classes: This support persistence of ozone objects on
+ datanode. These classes live under container directory.
+
+ 3. Client and Shell: We also support a ozone REST client lib, they are under
+ web/client and web/ozShell.
+
+ */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fbae8b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto
new file mode 100644
index 0000000..0fba636
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeContainerProtocol.proto
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and Unstable.
+ * Please see http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html
+ * for what changes are allowed for a *Unstable* .proto interface.
+ */
+
+// This file contains protocol buffers that are used to transfer data
+// to and from the datanode.
+option java_package = "org.apache.hadoop.hdfs.ozone.protocol.proto";
+option java_outer_classname = "ContainerProtos";
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs.ozone;
+import "hdfs.proto";
+
+/**
+ * Commands that are used to manipulate the state of containers on a datanode.
+ *
+ * These commands allow us to work against the datanode - from
+ * StorageContainer Manager as well as clients.
+ *
+ * 1. CreateContainer - This call is usually made by Storage Container
+ * manager, when we need to create a new container on a given datanode.
+ *
+ * 2. ReadContainer - Allows end user to stat a container. For example
+ * this allows us to return the metadata of a container.
+ *
+ * 3. UpdateContainer - Updates a container metadata.
+
+ * 4. DeleteContainer - This call is made to delete a container.
+ *
+ * 5. ListContainer - Returns the list of containers on this
+ * datanode. This will be used by tests and tools.
+ *
+ * 6. CreateKey - Given a valid container, creates a key.
+ *
+ * 7. ReadKey - Allows user to read the metadata of a Key.
+ *
+ * 8. UpdateKey - Updates the metadata of a Key.
+ *
+ * 9. DeleteKey - Deletes a given key.
+ *
+ * 10. ListKey - Returns a list of keys that are present inside
+ * a given container.
+ *
+ * 11. ReadChunk - Allows us to read a chunk.
+ *
+ * 12. DeleteChunk - Delete an unused chunk.
+ *
+ * 13. WriteChunk - Allows us to write a chunk
+ *
+ * 14. ListChunk - Given a Container/Key returns the list of Chunks.
+ *
+ */
+
+enum Type {
+ CreateContainer = 1;
+ ReadContainer = 2;
+ UpdateContainer = 3;
+ DeleteContainer = 4;
+ ListContainer = 5;
+
+ CreateKey = 6;
+ Readkey = 7;
+ UpdateKey = 8;
+ DeleteKey = 9;
+ ListKey = 10;
+
+ ReadChunk = 11;
+ DeleteChunk = 12;
+ WriteChunk = 13;
+ ListChunk = 14;
+}
+
+
+message ContainerCommandRequestProto {
+ required Type cmdType = 1; // Type of the command
+
+ // A string that identifies this command, we generate Trace ID in Ozone
+ // frontend and this allows us to trace that command all over ozone.
+ optional string traceID = 2;
+
+ // One of the following command is available when the corresponding
+ // cmdType is set. At the protocol level we allow only
+ // one command in each packet.
+ // TODO : Upgrade to Protobuf 2.6 or later.
+ optional CreateContainerRequestProto createContainer = 3;
+ optional ReadContainerRequestProto readContainer = 4;
+ optional UpdateContainerRequestProto updateContainer = 5;
+ optional DeleteContainerRequestProto deleteContainer = 6;
+ optional ListContainerRequestProto listContainer = 7;
+
+ optional CreateKeyRequestProto createKey = 8;
+ optional ReadKeyRequestProto readKey = 9;
+ optional UpdateKeyRequestProto updateKey = 10;
+ optional DeleteKeyRequestProto deleteKey = 11;
+ optional ListKeyRequestProto listKey = 12;
+
+ optional ReadChunkRequestProto readChunk = 13;
+ optional WriteChunkRequestProto writeChunk = 14;
+ optional DeleteChunkRequestProto deleteChunk = 15;
+ optional ListChunkRequestProto listChunk = 16;
+}
+
+message ContainerCommandResponseProto {
+ required Type cmdType = 1;
+ optional string traceID = 2;
+
+ optional CreateContainerResponseProto createContainer = 3;
+ optional ReadContainerResponseProto readContainer = 4;
+ optional UpdateContainerResponseProto updateContainer = 5;
+ optional DeleteContainerResponseProto deleteContainer = 6;
+ optional ListContainerResponseProto listContainer = 7;
+
+ optional CreateKeyResponseProto createKey = 8;
+ optional ReadKeyResponeProto readKey = 9;
+ optional UpdateKeyResponseProto updateKey = 10;
+ optional DeleteKeyResponeProto deleteKey = 11;
+ optional ListKeyResponeProto listKey = 12;
+
+ optional WriteChunkReponseProto writeChunk = 13;
+ optional ReadChunkReponseProto readChunk = 14;
+ optional DeleteChunkResponseProto deleteChunk = 15;
+ optional ListChunkResponseProto listChunk = 16;
+
+}
+
+// A pipeline is composed of one or more datanodes that back a container.
+message Pipeline {
+ required string leaderID = 1;
+ repeated DatanodeIDProto members = 2;
+ optional string containerName = 3;
+}
+
+message KeyValue {
+ required string key = 1;
+ optional string value = 2;
+}
+
+message ContainerData {
+ required string name = 1;
+ repeated KeyValue metadata = 2;
+ optional string containerPath = 3;
+}
+
+// Container Messages.
+message CreateContainerRequestProto {
+ required Pipeline pipeline = 1;
+ required ContainerData containerData = 2;
+}
+
+message CreateContainerResponseProto {
+}
+
+message ReadContainerRequestProto {
+ required Pipeline pipeline = 1;
+ required string name = 2;
+}
+
+message ReadContainerResponseProto {
+ optional ContainerData containerData = 2;
+}
+
+message UpdateContainerRequestProto {
+ required Pipeline pipeline = 1;
+ required ContainerData containerData = 2;
+}
+
+message UpdateContainerResponseProto {
+}
+
+message DeleteContainerRequestProto {
+ required Pipeline pipeline = 1;
+ required string name = 2;
+}
+
+message DeleteContainerResponseProto {
+}
+
+message ListContainerRequestProto {
+ required Pipeline pipeline = 1;
+ optional string prevKey = 2; // if this is not set query from start.
+ required uint32 count = 3; // Max Results to return
+}
+
+message ListContainerResponseProto {
+ repeated ContainerData containerData = 1;
+}
+
+
+message ContainerKeyData {
+ optional string containerName = 1;
+ required string name = 2;
+ repeated KeyValue metadata = 3;
+}
+
+// Key Messages.
+message CreateKeyRequestProto {
+ required Pipeline pipeline = 1;
+ required ContainerKeyData containerKeyData = 2;
+}
+
+message CreateKeyResponseProto {
+}
+
+message ReadKeyRequestProto {
+ required Pipeline pipeline = 1;
+ required ContainerKeyData containerKeyData = 2;
+}
+
+message ReadKeyResponeProto {
+ repeated KeyValue metadata = 1;
+ repeated chunkInfo chunkData = 2;
+}
+
+message UpdateKeyRequestProto {
+ required Pipeline pipeline = 1;
+ required ContainerKeyData containerKeyData = 2;
+}
+
+message UpdateKeyResponseProto {
+}
+
+
+message DeleteKeyRequestProto {
+ required Pipeline pipeline = 1;
+ required string name = 2;
+}
+
+message DeleteKeyResponeProto {
+}
+
+message ListKeyRequestProto {
+ required Pipeline pipeline = 1;
+ required string prevKey = 2;
+ required uint32 count = 3;
+}
+
+message ListKeyResponeProto {
+ repeated ContainerKeyData containerKeyData = 1;
+}
+
+// Chunk Operations
+
+message chunkInfo {
+ required uint64 offset = 1;
+ required uint64 len = 2;
+ optional uint64 checksum = 3;
+ repeated KeyValue metadata = 4;
+}
+
+message WriteChunkRequestProto {
+ required Pipeline pipeline = 1;
+ required string containerName = 2;
+ required string keyName = 3;
+ required chunkInfo chunkData = 4;
+ repeated bytes data = 5;
+}
+
+message WriteChunkReponseProto {
+}
+
+message ReadChunkRequestProto {
+ required Pipeline pipeline = 1;
+ required string containerName = 2;
+ required string keyName = 3;
+ required chunkInfo chunkData = 4;
+}
+
+message ReadChunkReponseProto {
+ required Pipeline pipeline = 1;
+ required chunkInfo chunkData = 2;
+ repeated bytes data = 3;
+}
+
+message DeleteChunkRequestProto {
+ required Pipeline pipeline = 1;
+ required string containerName = 2;
+ required string keyName = 3;
+ required chunkInfo chunkData = 4;
+}
+
+message DeleteChunkResponseProto {
+}
+
+message ListChunkRequestProto {
+ required Pipeline pipeline = 1;
+ required string containerName = 2;
+ required string keyName = 3;
+ required uint64 startOffset = 4;
+ required uint32 count = 5;
+}
+
+message ListChunkResponseProto {
+ repeated chunkInfo chunkData = 1;
+}
+