You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/02/15 00:27:56 UTC
atlas git commit: ATLAS-2446: updated HBase Model to capture more
attributes of table and column-family
Repository: atlas
Updated Branches:
refs/heads/master a2ccdf12c -> 19d67a132
ATLAS-2446: updated HBase Model to capture more attributes of table and column-family
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/19d67a13
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/19d67a13
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/19d67a13
Branch: refs/heads/master
Commit: 19d67a13206a68c49ce92926f9513e833c423c83
Parents: a2ccdf1
Author: rmani <rm...@hortonworks.com>
Authored: Wed Feb 14 14:43:16 2018 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Feb 14 16:25:04 2018 -0800
----------------------------------------------------------------------
.../atlas/hbase/bridge/HBaseAtlasHook.java | 56 ++++++
..._column_family_add_additional_attribute.json | 183 +++++++++++++++++++
2 files changed, 239 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/19d67a13/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
index f839f87..2074684 100644
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
+++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
@@ -92,6 +92,30 @@ public class HBaseAtlasHook extends AtlasHook {
public static final String ATTR_OWNER = "owner";
public static final String ATTR_NAME = "name";
+ // column addition metadata
+ public static final String ATTR_TABLE_MAX_FILESIZE = "maxFileSize";
+ public static final String ATTR_TABLE_ISREADONLY = "isReadOnly";
+ public static final String ATTR_TABLE_ISCOMPACTION_ENABLED = "isCompactionEnabled";
+ public static final String ATTR_TABLE_REPLICATION_PER_REGION = "replicasPerRegion";
+ public static final String ATTR_TABLE_DURABLILITY = "durability";
+
+ // column family additional metadata
+ public static final String ATTR_CF_BLOOMFILTER_TYPE = "bloomFilterType";
+ public static final String ATTR_CF_COMPRESSION_TYPE = "compressionType";
+ public static final String ATTR_CF_COMPACTION_COMPRESSION_TYPE = "compactionCompressionType";
+ public static final String ATTR_CF_ENCRYPTION_TYPE = "encryptionType";
+ public static final String ATTR_CF_KEEP_DELETE_CELLS = "keepDeletedCells";
+ public static final String ATTR_CF_MAX_VERSIONS = "maxVersions";
+ public static final String ATTR_CF_MIN_VERSIONS = "minVersions";
+ public static final String ATTR_CF_DATA_BLOCK_ENCODING = "dataBlockEncoding";
+ public static final String ATTR_CF_TTL = "ttl";
+ public static final String ATTR_CF_BLOCK_CACHE_ENABLED = "blockCacheEnabled";
+ public static final String ATTR_CF_CACHED_BLOOM_ON_WRITE = "cacheBloomsOnWrite";
+ public static final String ATTR_CF_CACHED_DATA_ON_WRITE = "cacheDataOnWrite";
+ public static final String ATTR_CF_CACHED_INDEXES_ON_WRITE = "cacheIndexesOnWrite";
+ public static final String ATTR_CF_EVICT_BLOCK_ONCLOSE = "evictBlocksOnClose";
+ public static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN = "prefetchBlocksOnOpen";
+
private static final String REFERENCEABLE_ATTRIBUTE_NAME = "qualifiedName";
private String clusterName = null;
@@ -422,12 +446,25 @@ public class HBaseAtlasHook extends AtlasHook {
table.setAttribute(ATTR_PARAMETERS, hbaseOperationContext.getHbaseConf());
table.setAttribute(ATTR_NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpace));
+ HTableDescriptor htableDescriptor = hbaseOperationContext.gethTableDescriptor();
+ if (htableDescriptor != null) {
+ table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize());
+ table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication());
+ table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.getMaxFileSize());
+ table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htableDescriptor.isCompactionEnabled());
+ table.setAttribute(ATTR_TABLE_DURABLILITY, (htableDescriptor.getDurability() != null ? htableDescriptor.getDurability().name() : null));
+ }
+
switch (operation) {
case CREATE_TABLE:
table.setAttribute(ATTR_CREATE_TIME, now);
table.setAttribute(ATTR_MODIFIED_TIME, now);
break;
+ case CREATE_COLUMN_FAMILY:
+ table.setAttribute(ATTR_MODIFIED_TIME, now);
+ break;
case ALTER_TABLE:
+ case ALTER_COLUMN_FAMILY:
table.setAttribute(ATTR_MODIFIED_TIME, now);
break;
default:
@@ -466,8 +503,27 @@ public class HBaseAtlasHook extends AtlasHook {
columnFamily.setAttribute(ATTR_OWNER, hbaseOperationContext.getOwner());
columnFamily.setAttribute(ATTR_TABLE, AtlasTypeUtil.getAtlasObjectId(table));
+ if (hColumnDescriptor!= null) {
+ columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hColumnDescriptor.isBlockCacheEnabled());
+ columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hColumnDescriptor.getBloomFilterType() != null ? hColumnDescriptor.getBloomFilterType().name():null));
+ columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite());
+ columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite());
+ columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, hColumnDescriptor.isCacheIndexesOnWrite());
+ columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (hColumnDescriptor.getCompactionCompressionType() != null ? hColumnDescriptor.getCompactionCompressionType().name():null));
+ columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, (hColumnDescriptor.getCompressionType() != null ? hColumnDescriptor.getCompressionType().name():null));
+ columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (hColumnDescriptor.getDataBlockEncoding() != null ? hColumnDescriptor.getDataBlockEncoding().name():null));
+ columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, hColumnDescriptor.getEncryptionType());
+ columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, hColumnDescriptor.isEvictBlocksOnClose());
+ columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( hColumnDescriptor.getKeepDeletedCells() != null ? hColumnDescriptor.getKeepDeletedCells().name():null));
+ columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, hColumnDescriptor.getMaxVersions());
+ columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, hColumnDescriptor.getMinVersions());
+ columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, hColumnDescriptor.isPrefetchBlocksOnOpen());
+ columnFamily.setAttribute(ATTR_CF_TTL, hColumnDescriptor.getTimeToLive());
+ }
+
switch (hbaseOperationContext.getOperation()) {
case CREATE_COLUMN_FAMILY:
+ case CREATE_TABLE:
columnFamily.setAttribute(ATTR_CREATE_TIME, now);
columnFamily.setAttribute(ATTR_MODIFIED_TIME, now);
break;
http://git-wip-us.apache.org/repos/asf/atlas/blob/19d67a13/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
new file mode 100644
index 0000000..0ee7908
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
@@ -0,0 +1,183 @@
+{
+ "patches": [
+ {
+ "action": "ADD_ATTRIBUTE",
+ "typeName": "hbase_table",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "maxFileSize",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isReadOnly",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isCompactionEnabled",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "replicasPerRegion",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "durability",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+
+ ]
+ },
+ {
+ "action": "ADD_ATTRIBUTE",
+ "typeName": "hbase_column_family",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "bloomFilterType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "compressionType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "compactionCompressionType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "encryptionType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "keepDeletedCells",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "maxVersions",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "minVersions",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "dataBlockEncoding",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "ttl",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "blockCacheEnabled",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "cacheBloomsOnWrite",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "cacheDataOnWrite",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "cacheIndexesOnWrite",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "evictBlocksOnClose",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "prefetchBlocksOnOpen",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file