You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2017/05/12 04:11:54 UTC
hadoop git commit: HDFS-11768. Ozone: KSM: Create Key Space manager
service. Contributed by Anu Engineer.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 1b1b74a09 -> 2796b3435
HDFS-11768. Ozone: KSM: Create Key Space manager service. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2796b343
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2796b343
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2796b343
Branch: refs/heads/HDFS-7240
Commit: 2796b3435e07eac0782da74bf636052d27f58222
Parents: 1b1b74a
Author: Anu Engineer <ae...@apache.org>
Authored: Thu May 11 21:11:32 2017 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu May 11 21:11:32 2017 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 2 +
.../apache/hadoop/ksm/helpers/VolumeArgs.java | 140 +++++++++
.../apache/hadoop/ksm/helpers/package-info.java | 18 ++
.../org/apache/hadoop/ksm/package-info.java | 21 ++
.../ksm/protocol/KeyspaceManagerProtocol.java | 98 +++++++
.../hadoop/ksm/protocol/package-info.java | 19 ++
.../protocolPB/KeySpaceManagerProtocolPB.java | 33 +++
.../hadoop/ksm/protocolPB/package-info.java | 19 ++
.../org/apache/hadoop/scm/ScmConfigKeys.java | 1 +
.../scm/container/common/helpers/Pipeline.java | 10 +-
.../hadoop/scm/storage/ChunkOutputStream.java | 2 +-
.../main/proto/DatanodeContainerProtocol.proto | 11 +-
.../main/proto/KeySpaceManagerProtocol.proto | 189 ++++++++++++
.../src/main/proto/Ozone.proto | 43 +++
.../StorageContainerLocationProtocol.proto | 2 +-
.../hadoop-hdfs/src/main/bin/hdfs | 7 +
.../apache/hadoop/ozone/OzoneClientUtils.java | 24 ++
.../container/common/helpers/ChunkInfo.java | 5 +-
.../container/common/helpers/ContainerData.java | 5 +-
.../ozone/container/common/helpers/KeyData.java | 5 +-
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 40 +++
.../hadoop/ozone/ksm/KeySpaceManager.java | 289 +++++++++++++++++++
.../apache/hadoop/ozone/ksm/package-info.java | 21 ++
.../hadoop/ozone/protocol/VersionResponse.java | 3 +-
...ceManagerProtocolServerSideTranslatorPB.java | 89 ++++++
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 2 +-
.../ozone/scm/container/ContainerMapping.java | 5 +-
.../web/storage/OzoneContainerTranslation.java | 2 +-
.../main/proto/CBlockClientServerProtocol.proto | 2 +-
.../StorageContainerDatanodeProtocol.proto | 2 +-
.../hadoop/ozone/TestOzoneClientUtils.java | 26 ++
.../ozone/container/ContainerTestHelper.java | 5 +-
32 files changed, 1108 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index f908d98..067cd7b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -190,6 +190,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>ReconfigurationProtocol.proto</include>
<include>StorageContainerLocationProtocol.proto</include>
<include>DatanodeContainerProtocol.proto</include>
+ <include>Ozone.proto</include>
+ <include>KeySpaceManagerProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java
new file mode 100644
index 0000000..472af65
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ksm.helpers;
+
+import com.google.common.base.Preconditions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A class that encapsulates the createVolume Args.
+ */
+public final class VolumeArgs {
+ private final String adminName;
+ private final String ownerName;
+ private final String volume;
+ private final long quotaInBytes;
+ private final Map<String, String> extendedAttributes;
+
+ /**
+ * Private constructor, constructed via builder.
+ *
+ * @param adminName - Administrator name.
+ * @param ownerName - Volume owner's name
+ * @param volume - volume name
+ * @param quotaInBytes - Volume Quota in bytes.
+ * @param keyValueMap - keyValue map.
+ */
+ private VolumeArgs(String adminName, String ownerName, String volume,
+ long quotaInBytes, Map<String, String> keyValueMap) {
+ this.adminName = adminName;
+ this.ownerName = ownerName;
+ this.volume = volume;
+ this.quotaInBytes = quotaInBytes;
+ this.extendedAttributes = keyValueMap;
+ }
+
+ /**
+ * Returns the Admin Name.
+ *
+ * @return String.
+ */
+ public String getAdminName() {
+ return adminName;
+ }
+
+ /**
+ * Returns the owner Name.
+ *
+ * @return String
+ */
+ public String getOwnerName() {
+ return ownerName;
+ }
+
+ /**
+ * Returns the volume Name.
+ *
+ * @return String
+ */
+ public String getVolume() {
+ return volume;
+ }
+
+ /**
+ * Returns Quota in Bytes.
+ *
+ * @return long, Quota in bytes.
+ */
+ public long getQuotaInBytes() {
+ return quotaInBytes;
+ }
+
+ public Map<String, String> getExtendedAttributes() {
+ return extendedAttributes;
+ }
+
+ static class Builder {
+ private String adminName;
+ private String ownerName;
+ private String volume;
+ private long quotaInBytes;
+ private Map<String, String> extendedAttributes;
+
+ /**
+ * Constructs a builder.
+ */
+ Builder() {
+ extendedAttributes = new HashMap<>();
+ }
+
+ public void setAdminName(String adminName) {
+ this.adminName = adminName;
+ }
+
+ public void setOwnerName(String ownerName) {
+ this.ownerName = ownerName;
+ }
+
+ public void setVolume(String volume) {
+ this.volume = volume;
+ }
+
+ public void setQuotaInBytes(long quotaInBytes) {
+ this.quotaInBytes = quotaInBytes;
+ }
+
+ public void addMetadata(String key, String value) {
+ extendedAttributes.put(key, value); // overwrite if present.
+ }
+
+ /**
+ * Constructs a CreateVolumeArgument.
+ *
+ * @return CreateVolumeArgs.
+ */
+ public VolumeArgs build() {
+ Preconditions.checkNotNull(adminName);
+ Preconditions.checkNotNull(ownerName);
+ Preconditions.checkNotNull(volume);
+ return new VolumeArgs(adminName, ownerName, volume, quotaInBytes,
+ extendedAttributes);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java
new file mode 100644
index 0000000..8e48773
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ksm.helpers;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java
new file mode 100644
index 0000000..5b208fa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ksm;
+/**
+ This package contains client side protocol library to communicate with KSM.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java
new file mode 100644
index 0000000..3a07d97
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ksm.protocol;
+
+import org.apache.hadoop.ksm.helpers.VolumeArgs;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Protocol to talk to KSM.
+ */
+public interface KeyspaceManagerProtocol {
+
+ /**
+ * Creates a volume.
+ * @param args - Arguments to create Volume.
+ * @throws IOException
+ */
+ void createVolume(VolumeArgs args) throws IOException;
+
+ /**
+ * Changes the owner of a volume.
+ * @param volume - Name of the volume.
+ * @param owner - Name of the owner.
+ * @throws IOException
+ */
+ void setOwner(String volume, String owner) throws IOException;
+
+ /**
+ * Changes the Quota on a volume.
+ * @param volume - Name of the volume.
+ * @param quota - Quota in bytes.
+ * @throws IOException
+ */
+ void setQuota(String volume, long quota) throws IOException;
+
+ /**
+ * Checks if the specified user can access this volume.
+ * @param volume - volume
+ * @param userName - user name
+ * @throws IOException
+ */
+ void checkVolumeAccess(String volume, String userName) throws IOException;
+
+ /**
+ * Gets the volume information.
+ * @param volume - Volume name.s
+ * @return VolumeArgs or exception is thrown.
+ * @throws IOException
+ */
+ VolumeArgs getVolumeinfo(String volume) throws IOException;
+
+ /**
+ * Deletes the an exisiting empty volume.
+ * @param volume - Name of the volume.
+ * @throws IOException
+ */
+ void deleteVolume(String volume) throws IOException;
+
+ /**
+ * Lists volume owned by a specific user.
+ * @param userName - user name
+ * @param prefix - Filter prefix -- Return only entries that match this.
+ * @param prevKey - Previous key -- List starts from the next from the prevkey
+ * @param maxKeys - Max number of keys to return.
+ * @return List of Volumes.
+ * @throws IOException
+ */
+ List<VolumeArgs> listVolumeByUser(String userName, String prefix, String
+ prevKey, long maxKeys) throws IOException;
+
+ /**
+ * Lists volume all volumes in the cluster.
+ * @param prefix - Filter prefix -- Return only entries that match this.
+ * @param prevKey - Previous key -- List starts from the next from the prevkey
+ * @param maxKeys - Max number of keys to return.
+ * @return List of Volumes.
+ * @throws IOException
+ */
+ List<VolumeArgs> listAllVolumes(String prefix, String
+ prevKey, long maxKeys) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java
new file mode 100644
index 0000000..d1d12dc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ksm.protocol;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java
new file mode 100644
index 0000000..1490008
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ksm.protocolPB;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyspaceManagerService;
+
+/**
+ * Protocol used to communicate with KSM.
+ */
+@ProtocolInfo(protocolName =
+ "org.apache.hadoop.ozone.protocol.KeyspaceManagerProtocol",
+ protocolVersion = 1)
+@InterfaceAudience.Private
+public interface KeySpaceManagerProtocolPB
+ extends KeyspaceManagerService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java
new file mode 100644
index 0000000..5dfa5ea
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ksm.protocolPB;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
index e779247..03f9c82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
@@ -152,6 +152,7 @@ public final class ScmConfigKeys {
"ozone.scm.container.provision_batch_size";
public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 10;
+
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
index 4456138..2ebc1e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.scm.container.common.helpers;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
@@ -60,7 +60,7 @@ public class Pipeline {
* @param pipeline - ProtoBuf definition for the pipeline.
* @return Pipeline Object
*/
- public static Pipeline getFromProtoBuf(ContainerProtos.Pipeline pipeline) {
+ public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) {
Preconditions.checkNotNull(pipeline);
Pipeline newPipeline = new Pipeline(pipeline.getLeaderID());
for (HdfsProtos.DatanodeIDProto dataID : pipeline.getMembersList()) {
@@ -104,9 +104,9 @@ public class Pipeline {
*
* @return Protobuf message
*/
- public ContainerProtos.Pipeline getProtobufMessage() {
- ContainerProtos.Pipeline.Builder builder =
- ContainerProtos.Pipeline.newBuilder();
+ public OzoneProtos.Pipeline getProtobufMessage() {
+ OzoneProtos.Pipeline.Builder builder =
+ OzoneProtos.Pipeline.newBuilder();
for (DatanodeID datanode : datanodes.values()) {
builder.addMembers(datanode.getProtoBufMessage());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
index 962ca6b..d93d2cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
@@ -30,7 +30,7 @@ import com.google.protobuf.ByteString;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.XceiverClientSpi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
index 2d018ee..012ff1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
@@ -29,6 +29,7 @@ option java_outer_classname = "ContainerProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs.ozone;
import "hdfs.proto";
+import "Ozone.proto";
/**
* Commands that are used to manipulate the state of containers on a datanode.
@@ -190,17 +191,7 @@ message ContainerCommandResponseProto {
}
-// A pipeline is composed of one or more datanodes that back a container.
-message Pipeline {
- required string leaderID = 1;
- repeated DatanodeIDProto members = 2;
- required string containerName = 3;
-}
-message KeyValue {
- required string key = 1;
- optional string value = 2;
-}
message ContainerData {
required string name = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
new file mode 100644
index 0000000..3ac25b8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
@@ -0,0 +1,189 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.ozone.protocol.proto";
+option java_outer_classname = "KeySpaceManagerProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs.ozone;
+
+/**
+This is file contains the protocol to communicate with
+Ozone key space manager. Ozone KSM manages the namespace for ozone.
+This is similar to Namenode for Ozone.
+*/
+
+import "Ozone.proto";
+
+enum Status {
+ OK = 1;
+ VOLUME_NOT_UNIQUE = 2;
+ VOLUME_NOT_FOUND = 3;
+ VOLUME_NOT_EMPTY = 4;
+ USER_NOT_FOUND = 5;
+ ACCESS_DENIED = 6;
+}
+
+
+message VolumeInfo {
+ required string adminName = 1;
+ required string ownerName = 2;
+ required string volume = 3;
+ optional uint64 quotaInBytes = 4;
+ repeated KeyValue metadata = 5;
+}
+
+/**
+ Creates a volume
+*/
+message CreateVolumeRequest {
+ required VolumeInfo volumeInfo = 1;
+}
+
+message CreateVolumeResponse {
+
+ required Status status = 1;
+}
+
+/**
+ Changes the Volume Properties -- like ownership and quota for a volume.
+*/
+message SetVolumePropertyRequest {
+ required string volumeName = 1;
+ optional string ownerName = 2;
+ optional uint64 quotaInBytes = 3;
+}
+
+message SetVolumePropertyResponse {
+ required Status status = 1;
+}
+
+/**
+ Checks if a specified user has access to the volume.
+*/
+message CheckVolumeAccessRequest {
+ required string volumeName = 1;
+ required string userName = 2;
+}
+
+message CheckVolumeAccessResponse {
+
+ required Status status = 1;
+}
+
+
+/**
+ Returns information about a volume.
+*/
+
+message InfoVolumeRequest {
+ required string volumeName = 1;
+}
+
+message InfoVolumeResponse {
+ required Status status = 1;
+ optional VolumeInfo volumeInfo = 2;
+
+}
+
+/**
+ Deletes an existing volume.
+*/
+message DeleteVolumeRequest {
+ required string volumeName = 1;
+}
+
+message DeleteVolumeResponse {
+ required Status status = 1;
+}
+
+
+/**
+ List Volumes -- List all volumes in the cluster or by user.
+*/
+
+message ListVolumeRequest {
+ enum Scope {
+ USER_VOLUMES = 1; // User volumes -- called by user
+ VOLUMES_BY_USER = 2; // User volumes - called by Admin
+ VOLUMES_BY_CLUSTER = 3; // All volumes in the cluster
+ }
+ required Scope scope = 1;
+ required string volumeName = 2;
+ optional string userName = 3;
+ optional string prefix = 4;
+ optional string prevKey = 5;
+ optional uint64 maxKeys = 6;
+}
+
+message ListVolumeResponse {
+ enum Status {
+ OK = 1;
+ ACCESS_DENIED = 2;
+ REQUIRED_ARG_MISSING = 3;
+ }
+ required Status status = 1;
+ repeated VolumeInfo volumeInfo = 2;
+}
+
+/**
+ The KSM service that takes care of Ozone namespace.
+*/
+service KeyspaceManagerService {
+
+ /**
+ Creates a Volume.
+ */
+ rpc createVolume(CreateVolumeRequest)
+ returns(CreateVolumeResponse);
+
+ /**
+ Allows modificiation of volume properties.
+ */
+ rpc setVolumeProperty(SetVolumePropertyRequest)
+ returns (SetVolumePropertyResponse);
+
+ /**
+ Checks if the specified volume is accesible by the specified user.
+ */
+ rpc checkVolumeAccess(CheckVolumeAccessRequest)
+ returns (CheckVolumeAccessResponse);
+
+ /**
+ Gets Volume information.
+ */
+ rpc infoVolume(InfoVolumeRequest)
+ returns(InfoVolumeResponse);
+ /**
+ Deletes a volume if it is empty.
+ */
+ rpc deleteVolume(DeleteVolumeRequest)
+ returns (DeleteVolumeResponse);
+
+ /**
+ Lists Volumes
+ */
+ rpc listVolumes(ListVolumeRequest)
+ returns (ListVolumeResponse);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
new file mode 100644
index 0000000..740038a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.ozone.protocol.proto";
+option java_outer_classname = "OzoneProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs.ozone;
+import "hdfs.proto";
+
+
+// A pipeline is composed of one or more datanodes that back a container.
+message Pipeline {
+ required string leaderID = 1;
+ repeated DatanodeIDProto members = 2;
+ required string containerName = 3;
+}
+
+message KeyValue {
+ required string key = 1;
+ optional string value = 2;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
index 8dfeb8b..772db2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
@@ -29,7 +29,7 @@ option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "hdfs.proto";
-import "DatanodeContainerProtocol.proto";
+import "Ozone.proto";
/**
* keys - batch of object keys to find
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 9d21f60..8ce29de 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -52,6 +52,7 @@ function hadoop_usage
hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "journalnode" "run the DFS journalnode"
hadoop_add_subcommand "jscsi" "run cblock jscsi server"
+ hadoop_add_subcommand "ksm" "Ozone keyspace manager"
hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs owned by the current user"
hadoop_add_subcommand "mover" "run a utility to move block replicas across storage types"
hadoop_add_subcommand "namenode" "run the DFS namenode"
@@ -164,6 +165,12 @@ function hdfscmd_case
jmxget)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
;;
+ ksm)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager
+ hadoop_debug "Appending HADOOP_KSM_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_KSM_OPTS}"
+ ;;
lsSnapshottableDir)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
index 908cc44..79275eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
@@ -35,6 +35,10 @@ import java.util.HashSet;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+ .OZONE_KSM_BIND_HOST_DEFAULT;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
@@ -224,6 +228,26 @@ public final class OzoneClientUtils {
port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
}
+
+ /**
+ * Retrieve the socket address that is used by KSM.
+ * @param conf
+ * @return Target InetSocketAddress for the SCM service endpoint.
+ */
+ public static InetSocketAddress getKsmAddress(
+ Configuration conf) {
+ final Optional<String> host = getHostNameFromConfigKeys(conf,
+ OZONE_KSM_ADDRESS_KEY);
+
+ // If no port number is specified then we'll just try the defaultBindPort.
+ final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
+ OZONE_KSM_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
+ host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" +
+ port.or(OZONE_KSM_PORT_DEFAULT));
+ }
+
/**
* Retrieve the hostname, trying the supplied config keys in order.
* Each config value may be absent, or if present in the format
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
index 7a3d449..690e3b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import java.io.IOException;
import java.util.Map;
@@ -110,8 +111,8 @@ public class ChunkInfo {
}
for (Map.Entry<String, String> entry : metadata.entrySet()) {
- ContainerProtos.KeyValue.Builder keyValBuilder =
- ContainerProtos.KeyValue.newBuilder();
+ OzoneProtos.KeyValue.Builder keyValBuilder =
+ OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
index 2f26649..1b70b1b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.helpers;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.util.Time;
import java.io.IOException;
@@ -113,8 +114,8 @@ public class ContainerData {
builder.setOpen(this.isOpen());
for (Map.Entry<String, String> entry : metadata.entrySet()) {
- ContainerProtos.KeyValue.Builder keyValBuilder =
- ContainerProtos.KeyValue.newBuilder();
+ OzoneProtos.KeyValue.Builder keyValBuilder =
+ OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
index 289dbe4..590513e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.helpers;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import java.io.IOException;
import java.util.Collections;
@@ -81,8 +82,8 @@ public class KeyData {
builder.setName(this.getKeyName());
builder.addAllChunks(this.chunks);
for (Map.Entry<String, String> entry : metadata.entrySet()) {
- ContainerProtos.KeyValue.Builder keyValBuilder =
- ContainerProtos.KeyValue.newBuilder();
+ OzoneProtos.KeyValue.Builder keyValBuilder =
+ OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
new file mode 100644
index 0000000..a23d47b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+
+/**
+ * KSM Constants.
+ */
+public final class KSMConfigKeys {
+ /**
+ * Never constructed.
+ */
+ private KSMConfigKeys() {
+ }
+
+
+ public static final String OZONE_KSM_HANDLER_COUNT_KEY =
+ "ozone.scm.handler.count.key";
+ public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 200;
+
+ public static final String OZONE_KSM_ADDRESS_KEY =
+ "ozone.ksm.address";
+ public static final String OZONE_KSM_BIND_HOST_DEFAULT =
+ "0.0.0.0";
+ public static final int OZONE_KSM_PORT_DEFAULT = 9862;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
new file mode 100644
index 0000000..7250885
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
@@ -0,0 +1,289 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ksm.helpers.VolumeArgs;
+import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol;
+import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB;
+import org.apache.hadoop.ozone.OzoneClientUtils;
+import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.ozone.protocolPB
+ .KeyspaceManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ozone.scm.StorageContainerManager;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+ .OZONE_KSM_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+ .OZONE_KSM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
+ .KeyspaceManagerService.newReflectiveBlockingService;
+import static org.apache.hadoop.util.ExitUtil.terminate;
+
+/**
+ * Ozone Keyspace manager is the metadata manager of ozone.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
+public class KeySpaceManager implements KeyspaceManagerProtocol {
+ // TODO: Support JMX
+ private static final Logger LOG =
+ LoggerFactory.getLogger(KeySpaceManager.class);
+
+ private final RPC.Server ksmRpcServer;
+ private final InetSocketAddress ksmRpcAddress;
+
+ public KeySpaceManager(OzoneConfiguration conf) throws IOException {
+ final int handlerCount = conf.getInt(OZONE_KSM_HANDLER_COUNT_KEY,
+ OZONE_KSM_HANDLER_COUNT_DEFAULT);
+
+ RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
+ ProtobufRpcEngine.class);
+
+ BlockingService ksmService = newReflectiveBlockingService(
+ new KeyspaceManagerProtocolServerSideTranslatorPB(this));
+ final InetSocketAddress ksmNodeRpcAddr = OzoneClientUtils.
+ getKsmAddress(conf);
+ ksmRpcServer = startRpcServer(conf, ksmNodeRpcAddr,
+ KeySpaceManagerProtocolPB.class, ksmService,
+ handlerCount);
+ ksmRpcAddress = updateListenAddress(conf,
+ OZONE_KSM_ADDRESS_KEY, ksmNodeRpcAddr, ksmRpcServer);
+
+ //TODO : Add call to register MXBean for JMX.
+ }
+
+ /**
+ * Starts an RPC server, if configured.
+ *
+ * @param conf configuration
+ * @param addr configured address of RPC server
+ * @param protocol RPC protocol provided by RPC server
+ * @param instance RPC protocol implementation instance
+ * @param handlerCount RPC server handler count
+ *
+ * @return RPC server
+ * @throws IOException if there is an I/O error while creating RPC server
+ */
+ private static RPC.Server startRpcServer(OzoneConfiguration conf,
+ InetSocketAddress addr, Class<?> protocol, BlockingService instance,
+ int handlerCount) throws IOException {
+ RPC.Server rpcServer = new RPC.Builder(conf)
+ .setProtocol(protocol)
+ .setInstance(instance)
+ .setBindAddress(addr.getHostString())
+ .setPort(addr.getPort())
+ .setNumHandlers(handlerCount)
+ .setVerbose(false)
+ .setSecretManager(null)
+ .build();
+
+ DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
+ return rpcServer;
+ }
+
+ /**
+ * Main entry point for starting KeySpaceManager.
+ *
+ * @param argv arguments
+ * @throws IOException if startup fails due to I/O error
+ */
+ public static void main(String[] argv) throws IOException {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class,
+ argv, LOG);
+ try {
+ KeySpaceManager ksm = new KeySpaceManager(new OzoneConfiguration());
+ ksm.start();
+ ksm.join();
+ } catch (Throwable t) {
+ LOG.error("Failed to start the KeyspaceManager.", t);
+ terminate(1, t);
+ }
+ }
+
+ /**
+ * Builds a message for logging startup information about an RPC server.
+ *
+ * @param description RPC server description
+ * @param addr RPC server listening address
+ * @return server startup message
+ */
+ private static String buildRpcServerStartMessage(String description,
+ InetSocketAddress addr) {
+ return addr != null ? String.format("%s is listening at %s",
+ description, addr.toString()) :
+ String.format("%s not started", description);
+ }
+
+ /**
+ * After starting an RPC server, updates configuration with the actual
+ * listening address of that server. The listening address may be different
+ * from the configured address if, for example, the configured address uses
+ * port 0 to request use of an ephemeral port.
+ *
+ * @param conf configuration to update
+ * @param rpcAddressKey configuration key for RPC server address
+ * @param addr configured address
+ * @param rpcServer started RPC server.
+ */
+ private static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
+ String rpcAddressKey, InetSocketAddress addr, RPC.Server rpcServer) {
+ InetSocketAddress listenAddr = rpcServer.getListenerAddress();
+ InetSocketAddress updatedAddr = new InetSocketAddress(
+ addr.getHostString(), listenAddr.getPort());
+ conf.set(rpcAddressKey,
+ listenAddr.getHostString() + ":" + listenAddr.getPort());
+ return updatedAddr;
+ }
+
+ /**
+ * Start service.
+ */
+ public void start() {
+ LOG.info(buildRpcServerStartMessage("KeyspaceManager RPC server",
+ ksmRpcAddress));
+ ksmRpcServer.start();
+ }
+
+ /**
+ * Wait until service has completed shutdown.
+ */
+ public void join() {
+ try {
+ ksmRpcServer.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.info("Interrupted during KeyspaceManager join.");
+ }
+ }
+
+ /**
+ * Creates a volume.
+ *
+ * @param args - Arguments to create Volume.
+ * @throws IOException
+ */
+ @Override
+ public void createVolume(VolumeArgs args) throws IOException {
+
+ }
+
+ /**
+ * Changes the owner of a volume.
+ *
+ * @param volume - Name of the volume.
+ * @param owner - Name of the owner.
+ * @throws IOException
+ */
+ @Override
+ public void setOwner(String volume, String owner) throws IOException {
+
+ }
+
+ /**
+ * Changes the Quota on a volume.
+ *
+ * @param volume - Name of the volume.
+ * @param quota - Quota in bytes.
+ * @throws IOException
+ */
+ @Override
+ public void setQuota(String volume, long quota) throws IOException {
+
+ }
+
+ /**
+ * Checks if the specified user can access this volume.
+ *
+ * @param volume - volume
+ * @param userName - user name
+ * @throws IOException
+ */
+ @Override
+ public void checkVolumeAccess(String volume, String userName) throws
+ IOException {
+
+ }
+
+ /**
+ * Gets the volume information.
+ *
+ * @param volume - Volume name.s
+ * @return VolumeArgs or exception is thrown.
+ * @throws IOException
+ */
+ @Override
+ public VolumeArgs getVolumeinfo(String volume) throws IOException {
+ return null;
+ }
+
+ /**
+ * Deletes the an exisiting empty volume.
+ *
+ * @param volume - Name of the volume.
+ * @throws IOException
+ */
+ @Override
+ public void deleteVolume(String volume) throws IOException {
+
+ }
+
+ /**
+ * Lists volume owned by a specific user.
+ *
+ * @param userName - user name
+ * @param prefix - Filter prefix -- Return only entries that match this.
+ * @param prevKey - Previous key -- List starts from the next from the
+ * prevkey
+ * @param maxKeys - Max number of keys to return.
+ * @return List of Volumes.
+ * @throws IOException
+ */
+ @Override
+ public List<VolumeArgs> listVolumeByUser(String userName, String prefix,
+ String prevKey, long maxKeys) throws IOException {
+ return null;
+ }
+
+ /**
+ * Lists volume all volumes in the cluster.
+ *
+ * @param prefix - Filter prefix -- Return only entries that match this.
+ * @param prevKey - Previous key -- List starts from the next from the
+ * prevkey
+ * @param maxKeys - Max number of keys to return.
+ * @return List of Volumes.
+ * @throws IOException
+ */
+ @Override
+ public List<VolumeArgs> listAllVolumes(String prefix, String prevKey, long
+ maxKeys) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
new file mode 100644
index 0000000..09d9f32
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+/*
+ This package contains the keyspace manager classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
index cf742ae..8d8f4f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,7 +19,7 @@
package org.apache.hadoop.ozone.protocol;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.ozone.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..0725d25
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol;
+import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB;
+import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
+
+/**
+ * This class is the server-side translator that forwards requests received on
+ * {@link org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB} to the
+ * KeyspaceManagerService server implementation.
+ */
+public class KeyspaceManagerProtocolServerSideTranslatorPB implements
+ KeySpaceManagerProtocolPB {
+ private final KeyspaceManagerProtocol impl;
+
+ /**
+ * Constructs an instance of the server handler.
+ *
+ * @param impl KeySpaceManagerProtocolPB
+ */
+ public KeyspaceManagerProtocolServerSideTranslatorPB(
+ KeyspaceManagerProtocol impl) {
+ this.impl = impl;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.CreateVolumeResponse createVolume(
+ RpcController controller, KeySpaceManagerProtocolProtos
+ .CreateVolumeRequest
+ request) throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.SetVolumePropertyResponse
+ setVolumeProperty(RpcController controller, KeySpaceManagerProtocolProtos
+ .SetVolumePropertyRequest request) throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse
+ checkVolumeAccess(RpcController controller, KeySpaceManagerProtocolProtos
+ .CheckVolumeAccessRequest request) throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.InfoVolumeResponse infoVolume(
+ RpcController controller,
+ KeySpaceManagerProtocolProtos.InfoVolumeRequest request)
+ throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.DeleteVolumeResponse deleteVolume(
+ RpcController controller, KeySpaceManagerProtocolProtos
+ .DeleteVolumeRequest
+ request) throws ServiceException {
+ return null;
+ }
+
+ @Override
+ public KeySpaceManagerProtocolProtos.ListVolumeResponse listVolumes(
+ RpcController controller,
+ KeySpaceManagerProtocolProtos.ListVolumeRequest request)
+ throws ServiceException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
index 8b94e96..d18aca8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
@@ -26,8 +26,8 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Pipeline;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.Pipeline;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
index cf0076c..c2b74c6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
@@ -19,7 +20,7 @@ package org.apache.hadoop.ozone.scm.container;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -172,7 +173,7 @@ public class ContainerMapping implements Mapping {
containerName);
}
pipeline = Pipeline.getFromProtoBuf(
- ContainerProtos.Pipeline.PARSER.parseFrom(pipelineBytes));
+ OzoneProtos.Pipeline.PARSER.parseFrom(pipelineBytes));
return pipeline;
} finally {
lock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java
index 9333fe6..18ade6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java
@@ -22,7 +22,7 @@ import java.util.List;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
index 1ec0d7f..0183bae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
@@ -27,7 +27,7 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.cblock;
-import "DatanodeContainerProtocol.proto";
+import "Ozone.proto";
/**
* This message is sent from CBlock client side to CBlock server to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
index db814c0..38c6dd1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -38,7 +38,7 @@ import "HdfsServer.proto";
import "DatanodeProtocol.proto";
-import "DatanodeContainerProtocol.proto";
+import "Ozone.proto";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
index aef53e9..a5dbea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -321,4 +322,29 @@ public class TestOzoneClientUtils {
assertTrue(e instanceof IllegalArgumentException);
}
}
+
+ @Test
+ public void testGetKSMAddress() {
+ final Configuration conf = new OzoneConfiguration();
+
+ // First try a client address with just a host name. Verify it falls
+ // back to the default port.
+ conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4");
+ InetSocketAddress addr = OzoneClientUtils.getKsmAddress(conf);
+ assertThat(addr.getHostString(), is("1.2.3.4"));
+ assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
+
+ // Next try a client address with just a host name and port. Verify the port
+ // is ignored and the default KSM port is used.
+ conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100");
+ addr = OzoneClientUtils.getKsmAddress(conf);
+ assertThat(addr.getHostString(), is("1.2.3.4"));
+ assertThat(addr.getPort(), is(100));
+
+ // Assert the we are able to use default configs if no value is specified.
+ conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "");
+ addr = OzoneClientUtils.getKsmAddress(conf);
+ assertThat(addr.getHostString(), is("0.0.0.0"));
+ assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2796b343/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
index 89b358a..89ad950 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
.ContainerCommandResponseProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -361,8 +362,8 @@ public final class ContainerTestHelper {
containerData.setName(containerName);
String[] keys = metaData.keySet().toArray(new String[]{});
for(int i=0; i<keys.length; i++) {
- ContainerProtos.KeyValue.Builder kvBuilder =
- ContainerProtos.KeyValue.newBuilder();
+ OzoneProtos.KeyValue.Builder kvBuilder =
+ OzoneProtos.KeyValue.newBuilder();
kvBuilder.setKey(keys[i]);
kvBuilder.setValue(metaData.get(keys[i]));
containerData.addMetadata(i, kvBuilder.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org