You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/12/04 16:04:09 UTC
[14/17] storm git commit: client blobstore interface documentation
update
client blobstore interface documentation update
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6687bedb
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6687bedb
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6687bedb
Branch: refs/heads/master
Commit: 6687bedbe86c1b04cecf18e02e6e2359ce2c3ead
Parents: 57240b4
Author: Sanket <sc...@untilservice-lm>
Authored: Mon Nov 30 18:02:39 2015 -0600
Committer: Sanket <sc...@untilservice-lm>
Committed: Mon Nov 30 18:02:39 2015 -0600
----------------------------------------------------------------------
.../storm/hdfs/blobstore/HdfsBlobStore.java | 14 +++
.../hdfs/blobstore/HdfsClientBlobStore.java | 5 +
.../storm/blobstore/ClientBlobStore.java | 126 ++++++++++++++++++-
.../storm/blobstore/LocalFsBlobStore.java | 15 +++
.../storm/blobstore/NimbusBlobStore.java | 8 ++
5 files changed, 166 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
index 144ad71..c344bcf 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
@@ -54,6 +54,20 @@ import static backtype.storm.blobstore.BlobStoreAclHandler.WRITE;
* Provides a HDFS file system backed blob store implementation.
* Note that this provides an api for having HDFS be the backing store for the blobstore,
* it is not a service/daemon.
+ *
+ * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN
+ * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus.
+ *
+ * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER
+ * who has read, write or admin privileges in order to perform respective operations on the blob.
+ *
+ * For hdfs blob store
+ * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. Here, unlike
+ * local blob store which stores the blobs locally, the nimbus talks to HDFS to upload the blobs.
+ * 2. The USER sets the ACLs, and the blob access is validated against these ACLs.
+ * 3. The SUPERVISOR interacts with nimbus thorugh HdfsClientBlobStore to download the blobs. Here, unlike local
+ * blob store the supervisor interacts with HDFS directly to download the blobs. The call to HdfsBlobStore is made as a "null"
+ * subject. The blobstore gets the hadoop user and validates permissions for the supervisor.
*/
public class HdfsBlobStore extends BlobStore {
public static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStore.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
index ec17dae..18c97da 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java
@@ -35,6 +35,11 @@ import java.util.Map;
/**
* Client to access the HDFS blobStore. At this point, this is meant to only be used by the
* supervisor. Don't trust who the client says they are so pass null for all Subjects.
+ *
+ * The HdfsBlobStore implementation takes care of the null Subjects. It assigns Subjects
+ * based on what hadoop says who the users are. These users must be configured accordingly
+ * in the SUPERVISOR_ADMINS for ACL validation and for the supervisors to download the blobs.
+ * This API is only used by the supervisor in order to talk directly to HDFS.
*/
public class HdfsClientBlobStore extends ClientBlobStore {
private static final Logger LOG = LoggerFactory.getLogger(HdfsClientBlobStore.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java
index cc40aff..6408469 100644
--- a/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java
+++ b/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java
@@ -28,22 +28,137 @@ import backtype.storm.utils.NimbusClient;
import java.util.Iterator;
import java.util.Map;
+/**
+ * The ClientBlobStore has two concrete implementations
+ * 1. NimbusBlobStore
+ * 2. HdfsClientBlobStore
+ *
+ * Create, update, read and delete are some of the basic operations defined by this interface.
+ * Each operation is validated for permissions against an user. We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS
+ * configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN access whereas the SUPERVISOR_ADMINS are given READ
+ * access in order to read and download the blobs form the nimbus.
+ *
+ * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER
+ * who has read, write or admin privileges in order to perform respective operations on the blob.
+ *
+ * For more detailed implementation
+ * @see backtype.storm.blobstore.NimbusBlobStore
+ * @see backtype.storm.blobstore.LocalFsBlobStore
+ * @see org.apache.storm.hdfs.blobstore.HdfsClientBlobStore
+ * @see org.apache.storm.hdfs.blobstore.HdfsBlobStore
+ */
public abstract class ClientBlobStore implements Shutdownable {
protected Map conf;
+ /**
+ * Sets up the client API by parsing the configs.
+ * @param conf The storm conf containing the config details.
+ */
public abstract void prepare(Map conf);
+
+ /**
+ * Client facing API to create a blob.
+ * @param key blob key name.
+ * @param meta contains ACL information.
+ * @return AtomicOutputStream returns an output stream into which data can be written.
+ * @throws AuthorizationException
+ * @throws KeyAlreadyExistsException
+ */
protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException;
+
+ /**
+ * Client facing API to update a blob.
+ * @param key blob key name.
+ * @return AtomicOutputStream returns an output stream into which data can be written.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
public abstract AtomicOutputStream updateBlob(String key) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to read the metadata information.
+ * @param key blob key name.
+ * @return AtomicOutputStream returns an output stream into which data can be written.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
public abstract ReadableBlobMeta getBlobMeta(String key) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to set the metadata for a blob.
+ * @param key blob key name.
+ * @param meta contains ACL information.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
protected abstract void setBlobMetaToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to delete a blob.
+ * @param key blob key name.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
public abstract void deleteBlob(String key) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to read a blob.
+ * @param key blob key name.
+ * @return an InputStream to read the metadata for a blob.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
public abstract InputStreamWithMeta getBlob(String key) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * @return Iterator for a list of keys currently present in the blob store.
+ */
public abstract Iterator<String> listKeys();
- public abstract int getBlobReplication(String Key) throws AuthorizationException, KeyNotFoundException;
- public abstract int updateBlobReplication(String Key, int replication) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to read the replication of a blob.
+ * @param key blob key name.
+ * @return int indicates the replication factor of a blob.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
+ public abstract int getBlobReplication(String key) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to update the replication of a blob.
+ * @param key blob key name.
+ * @param replication int indicates the replication factor a blob has to be set.
+ * @return int indicates the replication factor of a blob.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
+ public abstract int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException;
+
+ /**
+ * Client facing API to set a nimbus client.
+ * @param conf storm conf
+ * @param client NimbusClient
+ * @return indicates where the client connection has been setup.
+ */
public abstract boolean setClient(Map conf, NimbusClient client);
+
+ /**
+ * Creates state inside a zookeeper.
+ * Required for blobstore to write to zookeeper
+ * when Nimbus HA is turned on in order to maintain
+ * state consistency
+ * @param key
+ */
public abstract void createStateInZookeeper(String key);
+ /**
+ * Client facing API to create a blob.
+ * @param key blob key name.
+ * @param meta contains ACL information.
+ * @return AtomicOutputStream returns an output stream into which data can be written.
+ * @throws AuthorizationException
+ * @throws KeyAlreadyExistsException
+ */
public final AtomicOutputStream createBlob(String key, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException {
if (meta !=null && meta.is_set_acl()) {
BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
@@ -51,6 +166,13 @@ public abstract class ClientBlobStore implements Shutdownable {
return createBlobToExtend(key, meta);
}
+ /**
+ * Client facing API to set the metadata for a blob.
+ * @param key blob key name.
+ * @param meta contains ACL information.
+ * @throws AuthorizationException
+ * @throws KeyNotFoundException
+ */
public final void setBlobMeta(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException {
if (meta !=null && meta.is_set_acl()) {
BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java
index 0941b9a..ac7a4bd 100644
--- a/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java
@@ -48,6 +48,21 @@ import static backtype.storm.blobstore.BlobStoreAclHandler.WRITE;
/**
* Provides a local file system backed blob store implementation for Nimbus.
+ *
+ * For a local blob store the user and the supervisor use NimbusBlobStore Client API in order to talk to nimbus through thrift.
+ * The authentication and authorization here is based on the subject.
+ * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN
+ * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus.
+ *
+ * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER
+ * who has read, write or admin privileges in order to perform respective operations on the blob.
+ *
+ * For local blob store
+ * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API.
+ * 2. The USER sets the ACLs, and the blob access is validated against these ACLs.
+ * 3. The SUPERVISOR interacts with nimbus through the NimbusBlobStore Client API to download the blobs.
+ * The supervisors principal should match the set of users configured into SUPERVISOR_ADMINS.
+ * Here, the PrincipalToLocalPlugin takes care of mapping the principal to user name before the ACL validation.
*/
public class LocalFsBlobStore extends BlobStore {
public static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStore.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java
index bf084bb..334e6bb 100644
--- a/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java
+++ b/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java
@@ -37,6 +37,14 @@ import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
+/**
+ * NimbusBlobStore is a USER facing client API to perform
+ * basic operations such as create, update, delete and read
+ * for local and hdfs blob store.
+ *
+ * For local blob store it is also the client facing API for
+ * supervisor in order to download blobs from nimbus.
+ */
public class NimbusBlobStore extends ClientBlobStore {
private static final Logger LOG = LoggerFactory.getLogger(NimbusBlobStore.class);