You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/15 16:32:49 UTC
[34/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
deleted file mode 100644
index 85f8d6f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
deleted file mode 100644
index 93ed005..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.kubernetes;
-
-import com.google.gson.reflect.TypeToken;
-import com.squareup.okhttp.RequestBody;
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.apis.CoreV1Api;
-import io.kubernetes.client.models.V1ISCSIVolumeSource;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1ObjectReference;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import io.kubernetes.client.models.V1PersistentVolumeSpec;
-import io.kubernetes.client.util.Config;
-import io.kubernetes.client.util.Watch;
-import okio.Buffer;
-import org.apache.hadoop.cblock.cli.CBlockCli;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.storage.StorageManager;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CBLOCK_USER;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY;
-
-/**
- * Kubernetes Dynamic Persistent Volume provisioner.
- *
- * Listens on the kubernetes feed and creates the appropriate cblock AND
- * kubernetes PersistentVolume according to the created PersistentVolumeClaims.
- */
-public class DynamicProvisioner implements Runnable{
-
- protected static final Logger LOGGER =
- LoggerFactory.getLogger(DynamicProvisioner.class);
-
- private static final String STORAGE_CLASS = "cblock";
-
- private static final String PROVISIONER_ID = "hadoop.apache.org/cblock";
- private static final String KUBERNETES_PROVISIONER_KEY =
- "volume.beta.kubernetes.io/storage-provisioner";
- private static final String KUBERNETES_BIND_COMPLETED_KEY =
- "pv.kubernetes.io/bind-completed";
-
- private boolean running = true;
-
- private final StorageManager storageManager;
-
- private String kubernetesConfigFile;
-
- private String externalIp;
-
- private int externalPort;
-
- private String cblockUser;
-
- private CoreV1Api api;
-
- private ApiClient client;
-
- private Thread watcherThread;
-
- public DynamicProvisioner(OzoneConfiguration ozoneConf,
- StorageManager storageManager) throws IOException {
- this.storageManager = storageManager;
-
- kubernetesConfigFile = ozoneConf
- .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY);
-
- String jscsiServerAddress = ozoneConf
- .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
- DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
-
- externalIp = ozoneConf.
- getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
-
- externalPort = ozoneConf.
- getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
- DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
-
- cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER,
- DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT);
-
-
- }
-
- public void init() throws IOException {
- if (kubernetesConfigFile != null) {
- client = Config.fromConfig(kubernetesConfigFile);
- } else {
- client = Config.fromCluster();
- }
- client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS);
- Configuration.setDefaultApiClient(client);
- api = new CoreV1Api();
-
- watcherThread = new Thread(this);
- watcherThread.setName("DynamicProvisioner");
- watcherThread.setDaemon(true);
- }
-
- @Override
- public void run() {
- LOGGER.info("Starting kubernetes dynamic provisioner.");
- while (running) {
- String resourceVersion = null;
- try {
-
- Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client,
- api.listPersistentVolumeClaimForAllNamespacesCall(null,
- null,
- false,
- null,
- null,
- null,
- resourceVersion,
- null,
- true,
- null,
- null),
- new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() {
- }.getType());
-
-
- //check the new pvc resources, and create cblock + pv if needed
- for (Watch.Response<V1PersistentVolumeClaim> item : watch) {
- V1PersistentVolumeClaim claim = item.object;
-
- if (isPvMissingForPvc(claim)) {
-
- LOGGER.info("Provisioning volumes for PVC {}/{}",
- claim.getMetadata().getNamespace(),
- claim.getMetadata().getName());
-
- if (LOGGER.isDebugEnabled()) {
- RequestBody request =
- api.getApiClient().serialize(claim, "application/json");
-
- final Buffer buffer = new Buffer();
- request.writeTo(buffer);
- LOGGER.debug("New PVC is detected: " + buffer.readUtf8());
- }
-
- String volumeName = createVolumeName(claim);
-
- long size = CBlockCli.parseSize(
- claim.getSpec().getResources().getRequests().get("storage"));
-
- createCBlock(volumeName, size);
- createPersistentVolumeFromPVC(item.object, volumeName);
- }
- }
- } catch (Exception ex) {
- if (ex.getCause() != null && ex
- .getCause() instanceof SocketTimeoutException) {
- //This is normal. We are connection to the kubernetes server and the
- //connection should be reopened time to time...
- LOGGER.debug("Time exception occured", ex);
- } else {
- LOGGER.error("Error on provisioning persistent volumes.", ex);
- try {
- //we can try again in the main loop
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOGGER.error("Error on sleeping after an error.", e);
- }
- }
- }
- }
- }
-
- private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) {
-
- Map<String, String> annotations = claim.getMetadata().getAnnotations();
-
- return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS
- .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID
- .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes"
- .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY));
- }
-
- @VisibleForTesting
- protected String createVolumeName(V1PersistentVolumeClaim claim) {
- return claim.getMetadata().getName() + "-" + claim.getMetadata()
- .getUid();
- }
-
- public void stop() {
- running = false;
- try {
- watcherThread.join(60000);
- } catch (InterruptedException e) {
- LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e);
- }
- }
-
- private void createCBlock(String volumeName, long size)
- throws CBlockException {
-
- MountVolumeResponse mountVolumeResponse =
- storageManager.isVolumeValid(cblockUser, volumeName);
- if (!mountVolumeResponse.getIsValid()) {
- storageManager
- .createVolume(cblockUser, volumeName, size, 4 * 1024);
- }
- }
-
- private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim,
- String volumeName) throws ApiException, IOException {
-
- V1PersistentVolume v1PersistentVolume =
- persitenceVolumeBuilder(claim, volumeName);
-
- if (LOGGER.isDebugEnabled()) {
- RequestBody request =
- api.getApiClient().serialize(v1PersistentVolume, "application/json");
-
- final Buffer buffer = new Buffer();
- request.writeTo(buffer);
- LOGGER.debug("Creating new PV: " + buffer.readUtf8());
- }
- api.createPersistentVolume(v1PersistentVolume, null);
- }
-
- protected V1PersistentVolume persitenceVolumeBuilder(
- V1PersistentVolumeClaim claim,
- String volumeName) {
-
- V1PersistentVolume v1PersistentVolume = new V1PersistentVolume();
- v1PersistentVolume.setKind("PersistentVolume");
- v1PersistentVolume.setApiVersion("v1");
-
- V1ObjectMeta metadata = new V1ObjectMeta();
- metadata.setName(volumeName);
- metadata.setNamespace(claim.getMetadata().getNamespace());
- metadata.setAnnotations(new HashMap<>());
-
- metadata.getAnnotations()
- .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID);
-
- metadata.getAnnotations()
- .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS);
-
- v1PersistentVolume.setMetadata(metadata);
-
- V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec();
-
- spec.setCapacity(new HashMap<>());
- spec.getCapacity().put("storage",
- claim.getSpec().getResources().getRequests().get("storage"));
-
- spec.setAccessModes(new ArrayList<>());
- spec.getAccessModes().add("ReadWriteOnce");
-
- V1ObjectReference claimRef = new V1ObjectReference();
- claimRef.setName(claim.getMetadata().getName());
- claimRef.setNamespace(claim.getMetadata().getNamespace());
- claimRef.setKind(claim.getKind());
- claimRef.setApiVersion(claim.getApiVersion());
- claimRef.setUid(claim.getMetadata().getUid());
- spec.setClaimRef(claimRef);
-
- spec.persistentVolumeReclaimPolicy("Delete");
-
- V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource();
- iscsi.setIqn(cblockUser + ":" + volumeName);
- iscsi.setLun(0);
- iscsi.setFsType("ext4");
- String portal = externalIp + ":" + externalPort;
- iscsi.setTargetPortal(portal);
- iscsi.setPortals(new ArrayList<>());
- iscsi.getPortals().add(portal);
-
- spec.iscsi(iscsi);
- v1PersistentVolume.setSpec(spec);
- return v1PersistentVolume;
- }
-
-
- @VisibleForTesting
- protected CoreV1Api getApi() {
- return api;
- }
-
- public void start() {
- watcherThread.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
deleted file mode 100644
index 3ec5aab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains helper classes to run hadoop cluster in kubernetes
- * environment.
- */
-package org.apache.hadoop.cblock.kubernetes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
deleted file mode 100644
index 00064a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- *
- * The internal representation of a container maintained by CBlock server.
- * Include enough information to exactly identify a container for read/write
- * operation.
- *
- * NOTE that this class is work-in-progress. Depends on HDFS-7240 container
- * implementation. Currently only to allow testing.
- */
-public class ContainerDescriptor {
- private final String containerID;
- // the index of this container with in a volume
- // on creation, there may be no way to know the index of the container
- // as it is a volume specific information
- private int containerIndex;
- private Pipeline pipeline;
-
- public ContainerDescriptor(String containerID) {
- this.containerID = containerID;
- }
-
- public ContainerDescriptor(String containerID, int containerIndex) {
- this.containerID = containerID;
- this.containerIndex = containerIndex;
- }
-
- public void setContainerIndex(int idx) {
- this.containerIndex = idx;
- }
-
- public String getContainerID() {
- return containerID;
- }
-
- public void setPipeline(Pipeline pipeline) {
- this.pipeline = pipeline;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public int getContainerIndex() {
- return containerIndex;
- }
-
- public long getUtilization() {
- return 0;
- }
-
- public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() {
- CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder =
- CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
- builder.setContainerID(containerID);
- builder.setIndex(containerIndex);
- if (pipeline != null) {
- builder.setPipeline(pipeline.getProtobufMessage());
- }
- return builder.build();
- }
-
- public static ContainerDescriptor fromProtobuf(byte[] data)
- throws InvalidProtocolBufferException {
- CBlockClientServerProtocolProtos.ContainerIDProto id =
- CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data);
- return new ContainerDescriptor(id.getContainerID(),
- (int)id.getIndex());
- }
-
- @Override
- public int hashCode() {
- return containerID.hashCode()*37 + containerIndex;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof ContainerDescriptor) {
- ContainerDescriptor other = (ContainerDescriptor)o;
- return containerID.equals(other.containerID) &&
- containerIndex == other.containerIndex;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
deleted file mode 100644
index 4f5930d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The internal representation maintained by CBlock server as the info for
- * a volume. Contains the list of containers belonging to this volume.
- *
- * Many methods of this class is made such that the volume information (
- * including container list) can be easily transformed into a Json string
- * that can be stored/parsed from a persistent store for cblock server
- * persistence.
- *
- * This class is still work-in-progress.
- */
-public class VolumeDescriptor {
- // The main data structure is the container location map
- // other thing are mainly just information
-
- // since only one operation at a time is allowed, no
- // need to consider concurrency control here
-
- // key is container id
-
- private static final Logger LOG =
- LoggerFactory.getLogger(VolumeDescriptor.class);
-
- private ConcurrentHashMap<String, ContainerDescriptor> containerMap;
- private String userName;
- private int blockSize;
- private long volumeSize;
- private String volumeName;
- // this is essentially the ordered keys of containerMap
- // which is kind of redundant information. But since we
- // are likely to access it frequently based on ordering.
- // keeping this copy to avoid having to sort the key every
- // time
- private List<String> containerIdOrdered;
-
- /**
- * This is not being called explicitly, but this is necessary as
- * it will be called by the parse method implicitly when
- * reconstructing the object from json string. The get*() methods
- * and set*() methods are for the same purpose also.
- */
- public VolumeDescriptor() {
- this(null, null, 0, 0);
- }
-
- public VolumeDescriptor(String userName, String volumeName, long volumeSize,
- int blockSize) {
- this.containerMap = new ConcurrentHashMap<>();
- this.userName = userName;
- this.volumeName = volumeName;
- this.blockSize = blockSize;
- this.volumeSize = volumeSize;
- this.containerIdOrdered = new LinkedList<>();
- }
-
- public String getUserName() {
- return userName;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public void setVolumeSize(long volumeSize) {
- this.volumeSize = volumeSize;
- }
-
- public int getBlockSize() {
- return blockSize;
- }
-
- public void setBlockSize(int blockSize) {
- this.blockSize = blockSize;
- }
-
- public void setContainerIDs(ArrayList<String> containerIDs) {
- containerIdOrdered.addAll(containerIDs);
- }
-
- public void addContainer(ContainerDescriptor containerDescriptor) {
- containerMap.put(containerDescriptor.getContainerID(),
- containerDescriptor);
- }
-
-
- public HashMap<String, Pipeline> getPipelines() {
- HashMap<String, Pipeline> pipelines = new HashMap<>();
- for (Map.Entry<String, ContainerDescriptor> entry :
- containerMap.entrySet()) {
- pipelines.put(entry.getKey(), entry.getValue().getPipeline());
- }
- return pipelines;
- }
-
- public boolean isEmpty() {
- VolumeInfo info = getInfo();
- return info.getUsage() == 0;
- }
-
- public VolumeInfo getInfo() {
- // TODO : need to actually go through all containers of this volume and
- // ask for their utilization.
- long utilization = 0;
- for (Map.Entry<String, ContainerDescriptor> entry :
- containerMap.entrySet()) {
- utilization += entry.getValue().getUtilization();
- }
- return new VolumeInfo(this.userName, this.volumeName,
- this.volumeSize, this.blockSize,
- utilization * blockSize);
- }
-
- public String[] getContainerIDs() {
- //ArrayList<Long> ids = new ArrayList(containerMap.keySet());
- //return ids.toArray(new Long[ids.size()]);
- return containerIdOrdered.toArray(new String[containerIdOrdered.size()]);
- }
-
- public List<String> getContainerIDsList() {
- return new ArrayList<>(containerIdOrdered);
- }
-
- public List<Pipeline> getContainerPipelines() {
- Map<String, Pipeline> tmp = getPipelines();
- List<Pipeline> pipelineList = new LinkedList<>();
- for (String containerIDString : containerIdOrdered) {
- pipelineList.add(tmp.get(containerIDString));
- }
- return pipelineList;
- }
-
- @Override
- public String toString() {
- String string = "";
- string += "Username:" + userName + "\n";
- string += "VolumeName:" + volumeName + "\n";
- string += "VolumeSize:" + volumeSize + "\n";
- string += "blockSize:" + blockSize + "\n";
- string += "containerIds:" + containerIdOrdered + "\n";
- string += "containerIdsWithObject:" + containerMap.keySet();
- return string;
- }
-
- public CBlockClientServerProtocolProtos.MountVolumeResponseProto
- toProtobuf() {
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume =
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder();
- volume.setIsValid(true);
- volume.setVolumeName(volumeName);
- volume.setUserName(userName);
- volume.setVolumeSize(volumeSize);
- volume.setBlockSize(blockSize);
- for (String containerIDString : containerIdOrdered) {
- ContainerDescriptor containerDescriptor = containerMap.get(
- containerIDString);
- volume.addAllContainerIDs(containerDescriptor.toProtobuf());
- }
- return volume.build();
- }
-
- public static VolumeDescriptor fromProtobuf(byte[] data)
- throws InvalidProtocolBufferException {
- CBlockClientServerProtocolProtos.MountVolumeResponseProto volume =
- CBlockClientServerProtocolProtos.MountVolumeResponseProto
- .parseFrom(data);
- String userName = volume.getUserName();
- String volumeName = volume.getVolumeName();
- long volumeSize = volume.getVolumeSize();
- int blockSize = volume.getBlockSize();
- VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName,
- volumeName, volumeSize, blockSize);
- List<CBlockClientServerProtocolProtos.ContainerIDProto> containers
- = volume.getAllContainerIDsList();
-
- String[] containerOrdering = new String[containers.size()];
-
- for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto :
- containers) {
- ContainerDescriptor containerDescriptor = new ContainerDescriptor(
- containerProto.getContainerID(),
- (int)containerProto.getIndex());
- if(containerProto.hasPipeline()) {
- containerDescriptor.setPipeline(
- Pipeline.getFromProtoBuf(containerProto.getPipeline()));
- }
- volumeDescriptor.addContainer(containerDescriptor);
- containerOrdering[containerDescriptor.getContainerIndex()] =
- containerDescriptor.getContainerID();
- }
- volumeDescriptor.setContainerIDs(
- new ArrayList<>(Arrays.asList(containerOrdering)));
- return volumeDescriptor;
- }
-
- @Override
- public int hashCode() {
- return userName.hashCode()*37 + volumeName.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof VolumeDescriptor) {
- VolumeDescriptor other = (VolumeDescriptor)o;
- if (!userName.equals(other.getUserName()) ||
- !volumeName.equals(other.getVolumeName()) ||
- volumeSize != other.getVolumeSize() ||
- blockSize != other.getBlockSize()) {
- return false;
- }
- if (containerIdOrdered.size() != other.containerIdOrdered.size() ||
- containerMap.size() != other.containerMap.size()) {
- return false;
- }
- for (int i = 0; i<containerIdOrdered.size(); i++) {
- if (!containerIdOrdered.get(i).equals(
- other.containerIdOrdered.get(i))) {
- return false;
- }
- }
- return containerMap.equals(other.containerMap);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
deleted file mode 100644
index 7f50c41..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-/**
- * A wrapper class that represents the information about a volume. Used in
- * communication between CBlock client and CBlock server only.
- */
-public class VolumeInfo {
- private final String userName;
- private final String volumeName;
- private final long volumeSize;
- private final long blockSize;
- private final long usage;
-
- public VolumeInfo(String userName, String volumeName, long volumeSize,
- long blockSize, long usage) {
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.usage = usage;
- }
-
- // When listing volume, the usage will not be set.
- public VolumeInfo(String userName, String volumeName, long volumeSize,
- long blockSize) {
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.usage = -1;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public long getBlockSize() {
- return blockSize;
- }
-
- public long getUsage() {
- return usage;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- @Override
- public String toString() {
- return " userName:" + userName +
- " volumeName:" + volumeName +
- " volumeSize:" + volumeSize +
- " blockSize:" + blockSize +
- " (sizeInBlocks:" + volumeSize/blockSize + ")" +
- " usageInBlocks:" + usage;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
deleted file mode 100644
index a331d7a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
deleted file mode 100644
index a7d5d8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
deleted file mode 100644
index fc40cef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * The protocol that CBlock client side uses to talk to server side. CBlock
- * client is the point where a volume is mounted. All the actual volume IO
- * operations will go through CBlock client after the volume is mounted.
- *
- * When users mount a volume on CBlock client, CBlock client side uses this
- * protocol to send mount request to CBlock server.
- */
-public interface CBlockClientProtocol {
- MountVolumeResponse mountVolume(String userName, String volumeName)
- throws IOException;
-
- List<VolumeInfo> listVolumes() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
deleted file mode 100644
index bf00bc0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * CBlock uses a separate command line tool to send volume management
- * operations to CBlock server, including create/delete/info/list volumes. This
- * is the protocol used by the command line tool to send these requests and get
- * responses from CBlock server.
- */
-@InterfaceAudience.Private
-public interface CBlockServiceProtocol {
-
- void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws IOException;
-
- void deleteVolume(String userName, String volumeName,
- boolean force) throws IOException;
-
- VolumeInfo infoVolume(String userName,
- String volumeName) throws IOException;
-
- List<VolumeInfo> listVolume(String userName) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
deleted file mode 100644
index 70ccd90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.util.HashMap;
-import java.util.List;
-
-/**
- * The response message of mounting a volume. Including enough information
- * for the client to communicate (perform IO) with the volume containers
- * directly.
- */
-public class MountVolumeResponse {
- private final boolean isValid;
- private final String userName;
- private final String volumeName;
- private final long volumeSize;
- private final int blockSize;
- private List<Pipeline> containerList;
- private HashMap<String, Pipeline> pipelineMap;
-
- public MountVolumeResponse(boolean isValid, String userName,
- String volumeName, long volumeSize, int blockSize,
- List<Pipeline> containerList,
- HashMap<String, Pipeline> pipelineMap) {
- this.isValid = isValid;
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.containerList = containerList;
- this.pipelineMap = pipelineMap;
- }
-
- public boolean getIsValid() {
- return isValid;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public int getBlockSize() {
- return blockSize;
- }
-
- public List<Pipeline> getContainerList() {
- return containerList;
- }
-
- public HashMap<String, Pipeline> getPipelineMap() {
- return pipelineMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
deleted file mode 100644
index 33438ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
deleted file mode 100644
index 99f3110..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * This is the protocol CBlock client uses to talk to CBlock server.
- * CBlock client is the mounting point of a volume. When a user mounts a
- * volume, the cBlock client running on the local node will use this protocol
- * to talk to CBlock server to mount the volume.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockClientServerProtocolPB extends
- CBlockClientServerProtocolProtos
- .CBlockClientServerProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index bfe2130..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * The server side implementation of cblock client to server protocol.
- */
-@InterfaceAudience.Private
-public class CBlockClientServerProtocolServerSideTranslatorPB implements
- CBlockClientServerProtocolPB {
-
- private final CBlockClientProtocol impl;
-
- public CBlockClientServerProtocolServerSideTranslatorPB(
- CBlockClientProtocol impl) {
- this.impl = impl;
- }
-
- @Override
- public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume(
- RpcController controller,
- CBlockClientServerProtocolProtos.MountVolumeRequestProto request)
- throws ServiceException {
- String userName = request.getUserName();
- String volumeName = request.getVolumeName();
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder
- resp =
- CBlockClientServerProtocolProtos
- .MountVolumeResponseProto.newBuilder();
- try {
- MountVolumeResponse result = impl.mountVolume(userName, volumeName);
- boolean isValid = result.getIsValid();
- resp.setIsValid(isValid);
- if (isValid) {
- resp.setUserName(result.getUserName());
- resp.setVolumeName(result.getVolumeName());
- resp.setVolumeSize(result.getVolumeSize());
- resp.setBlockSize(result.getBlockSize());
- List<Pipeline> containers = result.getContainerList();
- HashMap<String, Pipeline> pipelineMap = result.getPipelineMap();
-
- for (int i=0; i<containers.size(); i++) {
- CBlockClientServerProtocolProtos.ContainerIDProto.Builder id =
- CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
- String containerName = containers.get(i).getContainerName();
- id.setContainerID(containerName);
- id.setIndex(i);
- if (pipelineMap.containsKey(containerName)) {
- id.setPipeline(pipelineMap.get(containerName).getProtobufMessage());
- }
- resp.addAllContainerIDs(id.build());
- }
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return resp.build();
- }
-
- @Override
- public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes(
- RpcController controller,
- CBlockClientServerProtocolProtos.ListVolumesRequestProto request)
- throws ServiceException {
- try {
- CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp =
- CBlockClientServerProtocolProtos.ListVolumesResponseProto
- .newBuilder();
- List<VolumeInfo> volumeInfos = impl.listVolumes();
- List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos =
- volumeInfos.stream().map(
- volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto
- .newBuilder().setUserName(volumeInfo.getUserName())
- .setBlockSize(volumeInfo.getBlockSize())
- .setVolumeName(volumeInfo.getVolumeName())
- .setVolumeSize(volumeInfo.getVolumeSize())
- .setUsage(volumeInfo.getUsage()).build())
- .collect(Collectors.toList());
- resp.addAllVolumeEntry(convertedInfos);
- return resp.build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
deleted file mode 100644
index 282d6cd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * Users use a independent command line tool to talk to CBlock server for
- * volume operations (create/delete/info/list). This is the protocol used by
- * the the command line tool to send these requests to CBlock server.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockServiceProtocolPB extends
- CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 8924a0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT;
-
-/**
- * Server side implementation of the protobuf service.
- */
-@InterfaceAudience.Private
-public class CBlockServiceProtocolServerSideTranslatorPB
- implements CBlockServiceProtocolPB {
-
- private final CBlockServiceProtocol impl;
- private static final Logger LOG =
- LoggerFactory.getLogger(
- CBlockServiceProtocolServerSideTranslatorPB.class);
-
- @Override
- public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.CreateVolumeRequestProto request)
- throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("createVolume called! volume size: " + request.getVolumeSize()
- + " block size: " + request.getBlockSize());
- }
- try {
- if (request.hasBlockSize()) {
- impl.createVolume(request.getUserName(), request.getVolumeName(),
- request.getVolumeSize(), request.getBlockSize());
- } else{
- impl.createVolume(request.getUserName(), request.getVolumeName(),
- request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT);
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return CBlockServiceProtocolProtos.CreateVolumeResponseProto
- .newBuilder().build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.DeleteVolumeRequestProto request)
- throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("deleteVolume called! volume name: " + request.getVolumeName()
- + " force:" + request.getForce());
- }
- try {
- if (request.hasForce()) {
- impl.deleteVolume(request.getUserName(), request.getVolumeName(),
- request.getForce());
- } else {
- impl.deleteVolume(request.getUserName(), request.getVolumeName(),
- false);
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return CBlockServiceProtocolProtos.DeleteVolumeResponseProto
- .newBuilder().build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.InfoVolumeRequestProto request
- ) throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("infoVolume called! volume name: " + request.getVolumeName());
- }
- CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp =
- CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder();
- CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto =
- CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
- VolumeInfo volumeInfo;
- try {
- volumeInfo = impl.infoVolume(request.getUserName(),
- request.getVolumeName());
- } catch (IOException e) {
- throw new ServiceException(e);
- }
-
- volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize());
- volumeInfoProto.setBlockSize(volumeInfo.getBlockSize());
- volumeInfoProto.setUsage(volumeInfo.getUsage());
- volumeInfoProto.setUserName(volumeInfo.getUserName());
- volumeInfoProto.setVolumeName(volumeInfo.getVolumeName());
- resp.setVolumeInfo(volumeInfoProto);
- return resp.build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.ListVolumeRequestProto request
- ) throws ServiceException {
- CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp =
- CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder();
- String userName = null;
- if (request.hasUserName()) {
- userName = request.getUserName();
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("list volume received for :" + userName);
- }
- List<VolumeInfo> volumes;
- try {
- volumes = impl.listVolume(userName);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- for (VolumeInfo volume : volumes) {
- CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto
- = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
- volumeEntryProto.setUserName(volume.getUserName());
- volumeEntryProto.setVolumeName(volume.getVolumeName());
- volumeEntryProto.setVolumeSize(volume.getVolumeSize());
- volumeEntryProto.setBlockSize(volume.getBlockSize());
- resp.addVolumeEntry(volumeEntryProto.build());
- }
- return resp.build();
- }
-
- public CBlockServiceProtocolServerSideTranslatorPB(
- CBlockServiceProtocol impl) {
- this.impl = impl;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
deleted file mode 100644
index 5e03a92..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
deleted file mode 100644
index 865f3b2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.meta.ContainerDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * This class maintains the key space of CBlock, more specifically, the
- * volume to container mapping. The core data structure
- * is a map from users to their volumes info, where volume info is a handler
- * to a volume, containing information for IO on that volume and a storage
- * client responsible for talking to the SCM.
- */
-public class StorageManager {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(StorageManager.class);
- private final ScmClient storageClient;
- private final int numThreads;
- private static final int MAX_THREADS =
- Runtime.getRuntime().availableProcessors() * 2;
- private static final int MAX_QUEUE_CAPACITY = 1024;
- private final String cblockId;
-
- /**
- * We will NOT have the situation where same kv pair getting
- * processed, but it is possible to have multiple kv pair being
- * processed at same time.
- *
- * So using just ConcurrentHashMap should be sufficient
- *
- * Again since currently same user accessing from multiple places
- * is not allowed, no need to consider concurrency of volume map
- * within one user
- */
- private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>>
- user2VolumeMap;
- // size of an underlying container.
- // TODO : assuming all containers are of the same size
- private long containerSizeB;
-
- public StorageManager(ScmClient storageClient,
- OzoneConfiguration ozoneConfig, String cblockId) throws IOException {
- this.storageClient = storageClient;
- this.user2VolumeMap = new ConcurrentHashMap<>();
- this.containerSizeB = storageClient.getContainerSize(null);
- this.numThreads =
- ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE,
- CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT);
- this.cblockId = cblockId;
- }
-
- /**
- * This call will put the volume into in-memory map.
- *
- * more specifically, make the volume discoverable on jSCSI server
- * and keep it's reference in-memory for look up.
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume,
- * @param volume a {@link VolumeDescriptor} object encapsulating the
- * information about the volume.
- */
- private void makeVolumeReady(String userName, String volumeName,
- VolumeDescriptor volume) {
- HashMap<String, VolumeDescriptor> userVolumes;
- if (user2VolumeMap.containsKey(userName)) {
- userVolumes = user2VolumeMap.get(userName);
- } else {
- userVolumes = new HashMap<>();
- user2VolumeMap.put(userName, userVolumes);
- }
- userVolumes.put(volumeName, volume);
- }
-
- /**
- * Called by CBlockManager to add volumes read from persistent store into
- * memory, need to contact SCM to setup the reference to the containers given
- * their id.
- *
- * Only for failover process where container meta info is read from
- * persistent store, and containers themselves are alive.
- *
- * TODO : Currently, this method is not being called as failover process
- * is not implemented yet.
- *
- * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating
- * the information about a volume.
- * @throws IOException when adding the volume failed. e.g. volume already
- * exist, or no more container available.
- */
- public synchronized void addVolume(VolumeDescriptor volumeDescriptor)
- throws IOException{
- String userName = volumeDescriptor.getUserName();
- String volumeName = volumeDescriptor.getVolumeName();
- LOGGER.info("addVolume:" + userName + ":" + volumeName);
- if (user2VolumeMap.containsKey(userName)
- && user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Volume already exist for "
- + userName + ":" + volumeName);
- }
- // the container ids are read from levelDB, setting up the
- // container handlers here.
- String[] containerIds = volumeDescriptor.getContainerIDs();
-
- for (String containerId : containerIds) {
- try {
- Pipeline pipeline = storageClient.getContainer(containerId);
- ContainerDescriptor containerDescriptor =
- new ContainerDescriptor(containerId);
- containerDescriptor.setPipeline(pipeline);
- volumeDescriptor.addContainer(containerDescriptor);
- } catch (IOException e) {
- LOGGER.error("Getting container failed! Container:{} error:{}",
- containerId, e);
- throw e;
- }
- }
- // now ready to put into in-memory map.
- makeVolumeReady(userName, volumeName, volumeDescriptor);
- }
-
- private class CreateContainerTask implements Runnable {
- private final VolumeDescriptor volume;
- private final int containerIdx;
- private final ArrayList<String> containerIds;
- private final AtomicInteger numFailed;
-
- CreateContainerTask(VolumeDescriptor volume, int containerIdx,
- ArrayList<String> containerIds,
- AtomicInteger numFailed) {
- this.volume = volume;
- this.containerIdx = containerIdx;
- this.containerIds = containerIds;
- this.numFailed = numFailed;
- }
-
- /**
- * When an object implementing interface <code>Runnable</code> is used
- * to create a thread, starting the thread causes the object's
- * <code>run</code> method to be called in that separately executing
- * thread.
- * <p>
- * The general contract of the method <code>run</code> is that it may
- * take any action whatsoever.
- *
- * @see Thread#run()
- */
- public void run() {
- ContainerDescriptor container = null;
- try {
- Pipeline pipeline = storageClient.createContainer(
- OzoneProtos.ReplicationType.STAND_ALONE,
- OzoneProtos.ReplicationFactor.ONE,
- KeyUtil.getContainerName(volume.getUserName(),
- volume.getVolumeName(), containerIdx), cblockId);
-
- container = new ContainerDescriptor(pipeline.getContainerName());
-
- container.setPipeline(pipeline);
- container.setContainerIndex(containerIdx);
- volume.addContainer(container);
- containerIds.set(containerIdx, container.getContainerID());
- } catch (Exception e) {
- numFailed.incrementAndGet();
- if (container != null) {
- LOGGER.error("Error creating container Container:{}:" +
- " index:{} error:{}", container.getContainerID(),
- containerIdx, e);
- } else {
- LOGGER.error("Error creating container.", e);
- }
- }
- }
- }
-
- private boolean createVolumeContainers(VolumeDescriptor volume) {
- ArrayList<String> containerIds = new ArrayList<>();
- ThreadPoolExecutor executor = new ThreadPoolExecutor(
- Math.min(numThreads, MAX_THREADS),
- MAX_THREADS, 1, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
- new ThreadPoolExecutor.CallerRunsPolicy());
-
- AtomicInteger numFailedCreates = new AtomicInteger(0);
- long allocatedSize = 0;
- int containerIdx = 0;
- while (allocatedSize < volume.getVolumeSize()) {
- // adding null to allocate space in ArrayList
- containerIds.add(containerIdx, null);
- Runnable task = new CreateContainerTask(volume, containerIdx,
- containerIds, numFailedCreates);
- executor.submit(task);
- allocatedSize += containerSizeB;
- containerIdx += 1;
- }
-
- // issue the command and then wait for it to finish
- executor.shutdown();
- try {
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
- } catch (InterruptedException e) {
- LOGGER.error("Error creating volume:{} error:{}",
- volume.getVolumeName(), e);
- executor.shutdownNow();
- Thread.currentThread().interrupt();
- }
-
- volume.setContainerIDs(containerIds);
- return numFailedCreates.get() == 0;
- }
-
- private void deleteContainer(String containerID, boolean force) {
- try {
- Pipeline pipeline = storageClient.getContainer(containerID);
- storageClient.deleteContainer(pipeline, force);
- } catch (Exception e) {
- LOGGER.error("Error deleting container Container:{} error:{}",
- containerID, e);
- }
- }
-
- private void deleteVolumeContainers(List<String> containers, boolean force)
- throws CBlockException {
- ThreadPoolExecutor executor = new ThreadPoolExecutor(
- Math.min(numThreads, MAX_THREADS),
- MAX_THREADS, 1, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
- new ThreadPoolExecutor.CallerRunsPolicy());
-
- for (String deleteContainer : containers) {
- if (deleteContainer != null) {
- Runnable task = () -> deleteContainer(deleteContainer, force);
- executor.submit(task);
- }
- }
-
- // issue the command and then wait for it to finish
- executor.shutdown();
- try {
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
- } catch (InterruptedException e) {
- LOGGER.error("Error deleting containers error:{}", e);
- executor.shutdownNow();
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Called by CBlock server when creating a fresh volume. The core
- * logic is adding needed information into in-memory meta data.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @param volumeSize the size of the volume.
- * @param blockSize the block size of the volume.
- * @throws CBlockException when the volume can not be created.
- */
- public synchronized void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws CBlockException {
- LOGGER.debug("createVolume:" + userName + ":" + volumeName);
- if (user2VolumeMap.containsKey(userName)
- && user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Volume already exist for "
- + userName + ":" + volumeName);
- }
- if (volumeSize < blockSize) {
- throw new CBlockException("Volume size smaller than block size? " +
- "volume size:" + volumeSize + " block size:" + blockSize);
- }
- VolumeDescriptor volume
- = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize);
- boolean success = createVolumeContainers(volume);
- if (!success) {
- // cleanup the containers and throw the exception
- deleteVolumeContainers(volume.getContainerIDsList(), true);
- throw new CBlockException("Error when creating volume:" + volumeName);
- }
- makeVolumeReady(userName, volumeName, volume);
- }
-
- /**
- * Called by CBlock server to delete a specific volume. Mainly
- * to check whether it can be deleted, and remove it from in-memory meta
- * data.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @param force if set to false, only delete volume it is empty, otherwise
- * throw exception. if set to true, delete regardless.
- * @throws CBlockException when the volume can not be deleted.
- */
- public synchronized void deleteVolume(String userName, String volumeName,
- boolean force) throws CBlockException {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Deleting non-exist volume "
- + userName + ":" + volumeName);
- }
- if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) {
- throw new CBlockException("Deleting a non-empty volume without force!");
- }
- VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName);
- deleteVolumeContainers(volume.getContainerIDsList(), force);
- if (user2VolumeMap.get(userName).size() == 0) {
- user2VolumeMap.remove(userName);
- }
- }
-
- /**
- * Called by CBlock server to get information of a specific volume.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @return a {@link VolumeInfo} object encapsulating the information of the
- * volume.
- * @throws CBlockException when the information can not be retrieved.
- */
- public synchronized VolumeInfo infoVolume(String userName, String volumeName)
- throws CBlockException {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Getting info for non-exist volume "
- + userName + ":" + volumeName);
- }
- return user2VolumeMap.get(userName).get(volumeName).getInfo();
- }
-
- /**
- * Called by CBlock server to check whether the given volume can be
- * mounted, i.e. whether it can be found in the meta data.
- *
- * return a {@link MountVolumeResponse} with isValid flag to indicate
- * whether the volume can be mounted or not.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume
- * @return a {@link MountVolumeResponse} object encapsulating whether the
- * volume is valid, and if yes, the requried information for client to
- * read/write the volume.
- */
- public synchronized MountVolumeResponse isVolumeValid(
- String userName, String volumeName) {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- // in the case of invalid volume, no need to set any value other than
- // isValid flag.
- return new MountVolumeResponse(false, null, null, 0, 0, null, null);
- }
- VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName);
- return new MountVolumeResponse(true, userName,
- volumeName, volume.getVolumeSize(), volume.getBlockSize(),
- volume.getContainerPipelines(), volume.getPipelines());
- }
-
- /**
- * Called by CBlock manager to list all volumes.
- *
- * @param userName the userName whose volume to be listed, if set to null,
- * all volumes will be listed.
- * @return a list of {@link VolumeDescriptor} representing all volumes
- * requested.
- */
- public synchronized List<VolumeDescriptor> getAllVolume(String userName) {
- ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>();
- if (userName == null) {
- for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry
- : user2VolumeMap.entrySet()) {
- allVolumes.addAll(entry.getValue().values());
- }
- } else {
- if (user2VolumeMap.containsKey(userName)) {
- allVolumes.addAll(user2VolumeMap.get(userName).values());
- }
- }
- return allVolumes;
- }
-
- /**
- * Only for testing the behavior of create/delete volumes.
- */
- @VisibleForTesting
- public VolumeDescriptor getVolume(String userName, String volumeName) {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- return null;
- }
- return user2VolumeMap.get(userName).get(volumeName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
deleted file mode 100644
index 4426e6d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
deleted file mode 100644
index beb9e32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
-
-/**
- * A simply class that generates key mappings. (e.g. from (userName, volumeName)
- * pair to a single string volumeKey.
- */
-public final class KeyUtil {
- public static String getVolumeKey(String userName, String volumeName) {
- return userName + ":" + volumeName;
- }
-
- public static String getContainerName(String userName, String volumeName,
- int containerID) {
- return getVolumeKey(userName, volumeName) + "#" + containerID;
- }
-
- public static String getUserNameFromVolumeKey(String key) {
- return key.split(":")[0];
- }
-
- public static String getVolumeFromVolumeKey(String key) {
- return key.split(":")[1];
- }
-
- public static boolean isValidVolumeKey(String key) {
- return key.contains(":");
- }
-
- private KeyUtil() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
deleted file mode 100644
index 5b9aa0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 72f39fc..4c94e38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -36,8 +36,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
@@ -73,7 +71,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -1497,23 +1494,6 @@ public class DFSUtil {
return password;
}
- public static boolean isOzoneEnabled(Configuration conf) {
- String securityEnabled = conf.get(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- boolean securityAuthorizationEnabled = conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false);
-
- if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
- LOG.error("Ozone is not supported in a security enabled cluster. ");
- return false;
- } else {
- return conf.getBoolean(OZONE_ENABLED,
- OZONE_ENABLED_DEFAULT);
- }
- }
-
/**
* Converts a Date into an ISO-8601 formatted datetime string.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org