You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/03/13 08:43:09 UTC
[atlas] branch master updated: ATLAS-2987: Update component
versions of Atlas to use Hadoop3, HBase2 and Solr7
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 2e1c563 ATLAS-2987: Update component versions of Atlas to use Hadoop3, HBase2 and Solr7
2e1c563 is described below
commit 2e1c563418afbe4c1ac61b44b18d02c02c17235d
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Wed Mar 13 01:41:28 2019 -0700
ATLAS-2987: Update component versions of Atlas to use Hadoop3, HBase2 and Solr7
---
LICENSE | 3 +
NOTICE | 18 +-
addons/falcon-bridge-shim/pom.xml | 4 -
addons/falcon-bridge/pom.xml | 4 -
.../org/apache/atlas/falcon/hook/FalconHookIT.java | 2 +-
addons/hbase-bridge-shim/pom.xml | 4 +
.../atlas/hbase/hook/HBaseAtlasCoprocessor.java | 3458 +-------------------
addons/hbase-bridge/pom.xml | 93 +-
.../apache/atlas/hbase/bridge/HBaseAtlasHook.java | 184 +-
.../org/apache/atlas/hbase/bridge/HBaseBridge.java | 62 +-
.../atlas/hbase/hook/HBaseAtlasCoprocessor.java | 117 +-
.../hbase/hook/HBaseAtlasCoprocessorBase.java | 991 ------
.../atlas/hbase/model/HBaseOperationContext.java | 68 +-
.../org/apache/atlas/hbase/HBaseAtlasHookIT.java | 15 +-
.../src/test/resources/atlas-log4j.xml | 20 +-
addons/hbase-testing-util/pom.xml | 203 ++
.../atlas/hbase/TestHBaseTestingUtilSpinup.java | 59 +
.../src/test/resources/atlas-log4j.xml | 7 -
addons/hive-bridge-shim/pom.xml | 5 -
addons/hive-bridge/pom.xml | 54 +-
.../java/org/apache/atlas/hive/HiveITBase.java | 1 -
.../org/apache/atlas/hive/hook/HiveHookIT.java | 66 +-
.../hive-bridge/src/test/resources/hive-site.xml | 23 +-
addons/kafka-bridge/pom.xml | 2 +-
...ble_column_family_add_additional_attribute.json | 48 +
addons/sqoop-bridge-shim/pom.xml | 4 -
addons/sqoop-bridge/pom.xml | 10 +-
addons/storm-bridge-shim/pom.xml | 4 -
addons/storm-bridge/pom.xml | 9 +-
.../test/resources/atlas-application.properties | 2 +-
.../src/main/resources/checkstyle-suppressions.xml | 3 +
common/pom.xml | 7 +-
distro/pom.xml | 5 +-
distro/src/bin/atlas_config.py | 21 +-
distro/src/conf/atlas-env.sh | 2 +-
distro/src/conf/solr/schema.xml | 1 +
distro/src/conf/solr/solrconfig.xml | 1 +
distro/src/main/assemblies/standalone-package.xml | 15 +
graphdb/janus-hbase2/pom.xml | 75 +
.../janusgraph/diskstorage/hbase2/AdminMask.java | 74 +
.../diskstorage/hbase2/ConnectionMask.java | 55 +
.../diskstorage/hbase2/HBaseAdmin2_0.java | 167 +
.../janusgraph/diskstorage/hbase2/HBaseCompat.java | 58 +
.../diskstorage/hbase2/HBaseCompat2_0.java | 61 +
.../diskstorage/hbase2/HBaseCompatLoader.java | 90 +
.../hbase2/HBaseKeyColumnValueStore.java | 384 +++
.../diskstorage/hbase2/HBaseStoreManager.java | 986 ++++++
.../diskstorage/hbase2/HBaseTransaction.java | 31 +
.../diskstorage/hbase2/HConnection2_0.java | 58 +
.../janusgraph/diskstorage/hbase2/HTable2_0.java | 60 +
.../janusgraph/diskstorage/hbase2/TableMask.java | 45 +
graphdb/janus/pom.xml | 6 +
.../graphdb/janus/AtlasJanusGraphDatabase.java | 23 +
.../janus/graphson/AtlasElementPropertyConfig.java | 2 -
.../graphdb/janus/graphson/AtlasGraphSONMode.java | 2 -
.../janus/graphson/AtlasGraphSONTokens.java | 3 -
.../janus/graphson/AtlasGraphSONUtility.java | 2 -
.../janusgraph/diskstorage/solr/Solr6Index.java | 79 +-
graphdb/pom.xml | 1 +
intg/pom.xml | 10 +
.../org/apache/atlas/ApplicationProperties.java | 71 +
pom.xml | 62 +-
repository/pom.xml | 26 +-
.../audit/HBaseBasedAuditRepository.java | 239 +-
.../repository/graph/GraphBackedSearchIndexer.java | 2 +
.../store/graph/v2/EntityGraphRetriever.java | 37 +-
.../atlas/util/AtlasRepositoryConfiguration.java | 40 +-
server-api/pom.xml | 4 +
shaded/hbase-client-shaded/pom.xml | 86 -
shaded/hbase-server-shaded/pom.xml | 112 -
test-tools/src/main/resources/solr/solr.xml | 4 -
tools/atlas-migration-exporter/README | 54 +
.../{src/main/resources => }/atlas-log4j.xml | 0
.../main/resources => }/atlas_migration_export.py | 2 +-
tools/atlas-migration-exporter/pom.xml | 65 -
.../java/org/apache/atlas/migration/Exporter.java | 180 -
.../apache/atlas/migration/NoOpNotification.java | 54 -
.../migration/NoOpNotificationChangeListener.java | 71 -
.../src/main/resources/README | 37 -
.../src/main/resources/migrationContext.xml | 40 -
webapp/pom.xml | 34 +-
.../atlas/classification/InterfaceAudience.java | 48 -
.../web/filters/AtlasAuthenticationFilter.java | 4 +-
.../service/AtlasZookeeperSecurityProperties.java | 2 +-
.../filters/AtlasAuthenticationSimpleFilterIT.java | 4 +-
.../web/security/NegativeSSLAndKerberosTest.java | 2 +-
86 files changed, 3418 insertions(+), 5729 deletions(-)
diff --git a/LICENSE b/LICENSE
index 0480ae4..cdd4b9e 100755
--- a/LICENSE
+++ b/LICENSE
@@ -218,6 +218,9 @@ Apache License. For details, see 3party-licenses/janusgraph-LICENSE
This product bundles pnotify, which is available under
Apache License. For details, see 3party-licenses/pnotify-LICENSE
+This product bundles hppc, which is available under
+Apache License. For details, see 3party-licenses/pnotify-LICENSE
+
This product bundles mock(for python tests) 1.0.1, which is available under
BSD License. For details, see 3party-licenses/mock-LICENSE
diff --git a/NOTICE b/NOTICE
index 3937b11..93104f7 100755
--- a/NOTICE
+++ b/NOTICE
@@ -1,22 +1,6 @@
-Apache Atlas (incubating)
+Apache Atlas
Copyright [2015-2017] The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
-
-==============================================================
-
-This product bundles titan 0.5.4(https://github.com/thinkaurelius/titan/blob/titan05):
-
-==============================================================
- Titan: Distributed Graph Database
- Copyright 2012 and onwards Aurelius
-==============================================================
-Titan includes software developed by Aurelius (http://thinkaurelius.com/) and the following individuals:
-
- * Matthias Broecheler
- * Dan LaRocque
- * Marko A. Rodriguez
- * Stephen Mallette
- * Pavel Yaskevich
diff --git a/addons/falcon-bridge-shim/pom.xml b/addons/falcon-bridge-shim/pom.xml
index 4ea5df9..649e29d 100755
--- a/addons/falcon-bridge-shim/pom.xml
+++ b/addons/falcon-bridge-shim/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Atlas Falcon Bridge Shim</name>
<packaging>jar</packaging>
- <properties>
- <falcon.version>0.8</falcon.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index c399383..eeef506 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Atlas Falcon Bridge</name>
<packaging>jar</packaging>
- <properties>
- <falcon.version>0.8</falcon.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
index 05214e5..24f3616 100644
--- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
@@ -109,7 +109,7 @@ public class FalconHookIT {
break;
case PROCESS:
- ((org.apache.falcon.entity.v0.process.Process) entity).setName(name);
+ ((Process) entity).setName(name);
break;
}
return (T)entity;
diff --git a/addons/hbase-bridge-shim/pom.xml b/addons/hbase-bridge-shim/pom.xml
index d45b6a5..280dc4c 100644
--- a/addons/hbase-bridge-shim/pom.xml
+++ b/addons/hbase-bridge-shim/pom.xml
@@ -46,6 +46,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/addons/hbase-bridge-shim/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java b/addons/hbase-bridge-shim/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
index e8cb20b..0b69104 100755
--- a/addons/hbase-bridge-shim/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
+++ b/addons/hbase-bridge-shim/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
@@ -18,86 +18,39 @@
*/
package org.apache.atlas.hbase.hook;
-import java.io.IOException;
-import java.util.List;
-import java.util.NavigableSet;
import org.apache.atlas.plugin.classloader.AtlasPluginClassLoader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellScanner;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.MasterObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
-import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
-import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WALEntry;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
-import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.CleanupBulkLoadRequest;
-import org.apache.hadoop.hbase.protobuf.generated.SecureBulkLoadProtos.PrepareBulkLoadRequest;
-import org.apache.hadoop.hbase.regionserver.DeleteTracker;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
-import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
-import org.apache.hadoop.hbase.regionserver.Region;
-import org.apache.hadoop.hbase.regionserver.Region.Operation;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.StoreFile.Reader;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.wal.WALKey;
-import com.google.common.collect.ImmutableList;
-import java.util.Set;
-import com.google.common.net.HostAndPort;
+
+import java.io.IOException;
+import java.util.Optional;
-public class HBaseAtlasCoprocessor implements MasterObserver, RegionObserver, RegionServerObserver, BulkLoadObserver {
+public class HBaseAtlasCoprocessor implements MasterCoprocessor, MasterObserver, RegionObserver, RegionServerObserver {
public static final Log LOG = LogFactory.getLog(HBaseAtlasCoprocessor.class);
private static final String ATLAS_PLUGIN_TYPE = "hbase";
private static final String ATLAS_HBASE_HOOK_IMPL_CLASSNAME = "org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor";
- private AtlasPluginClassLoader atlasPluginClassLoader = null;
- private Object impl = null;
- private MasterObserver implMasterObserver = null;
- private RegionObserver implRegionObserver = null;
- private RegionServerObserver implRegionServerObserver = null;
- private BulkLoadObserver implBulkLoadObserver = null;
+ private AtlasPluginClassLoader atlasPluginClassLoader = null;
+ private Object impl = null;
+ private MasterObserver implMasterObserver = null;
+ private RegionObserver implRegionObserver = null;
+ private RegionServerObserver implRegionServerObserver = null;
+ private MasterCoprocessor implMasterCoprocessor = null;
public HBaseAtlasCoprocessor() {
if(LOG.isDebugEnabled()) {
@@ -128,7 +81,7 @@ public class HBaseAtlasCoprocessor implements MasterObserver, RegionObserver, Re
implMasterObserver = (MasterObserver)impl;
implRegionObserver = (RegionObserver)impl;
implRegionServerObserver = (RegionServerObserver)impl;
- implBulkLoadObserver = (BulkLoadObserver)impl;
+ implMasterCoprocessor = (MasterCoprocessor)impl;
} catch (Exception e) {
// check what need to be done
@@ -142,3462 +95,183 @@ public class HBaseAtlasCoprocessor implements MasterObserver, RegionObserver, Re
}
}
-
-
- @Override
- public void postScannerClose(ObserverContext<RegionCoprocessorEnvironment> c, InternalScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postScannerClose()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postScannerClose(c, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postScannerClose()");
- }
- }
-
- @Override
- public RegionScanner postScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, RegionScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postScannerOpen()");
- }
-
- final RegionScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postScannerOpen(c, scan, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postStartMaster()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postStartMaster(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postStartMaster()");
- }
-
- }
-
- @Override
- public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, HColumnDescriptor column) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preAddColumn()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preAddColumn(c, tableName, column);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preAddColumn()");
- }
- }
-
- @Override
- public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append append) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preAppend()");
- }
-
- final Result ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preAppend(c, append);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preAppend()");
- }
-
- return ret;
- }
-
- @Override
- public void preAssign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preAssign()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preAssign(c, regionInfo);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preAssign()");
- }
- }
-
- @Override
- public void preBalance(ObserverContext<MasterCoprocessorEnvironment> c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preBalance()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preBalance(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preBalance()");
- }
- }
-
- @Override
- public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> c, boolean newValue) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preBalanceSwitch()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implMasterObserver.preBalanceSwitch(c, newValue);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preBalanceSwitch()");
- }
-
- return ret;
- }
-
- @Override
- public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx, List<Pair<byte[], String>> familyPaths) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preBulkLoadHFile()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preBulkLoadHFile(ctx, familyPaths);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preBulkLoadHFile()");
- }
-
- }
-
- @Override
- public boolean preCheckAndDelete(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp, ByteArrayComparable comparator, Delete delete, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCheckAndDelete()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCheckAndDelete(c, row, family, qualifier, compareOp, comparator, delete, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCheckAndDelete()");
- }
-
- return ret;
- }
-
- @Override
- public boolean preCheckAndPut(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp, ByteArrayComparable comparator, Put put, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCheckAndPut()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCheckAndPut(c, row, family, qualifier, compareOp, comparator, put, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCheckAndPut()");
- }
-
- return ret;
- }
-
@Override
- public void preCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCloneSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preCloneSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCloneSnapshot()");
- }
+ public Optional<MasterObserver> getMasterObserver() {
+ return Optional.<MasterObserver>of(this);
}
@Override
- public void preClose(ObserverContext<RegionCoprocessorEnvironment> e, boolean abortRequested) throws IOException {
+ public void start(CoprocessorEnvironment env) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preClose()");
+ LOG.debug("==> HBaseAtlasCoprocessor.start()");
}
try {
activatePluginClassLoader();
- implRegionObserver.preClose(e, abortRequested);
+ if (env instanceof MasterCoprocessorEnvironment) {
+ implMasterCoprocessor.start(env);
+ }
} finally {
deactivatePluginClassLoader();
}
-
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preClose()");
+ LOG.debug("<== HBaseAtlasCoprocessor.start()");
}
}
@Override
- public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, InternalScanner scanner, ScanType scanType) throws IOException {
+ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc, RegionInfo[] regions) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompact()");
+ LOG.debug("==> HBaseAtlasCoprocessor.postCreateTable()");
}
- final InternalScanner ret;
-
try {
activatePluginClassLoader();
- ret = implRegionObserver.preCompact(e, store, scanner, scanType);
+ implMasterObserver.postCreateTable(ctx, desc, regions);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompact()");
+ LOG.debug("<== HBaseAtlasCoprocessor.postCreateTable()");
}
-
- return ret;
}
@Override
- public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> e, Store store, List<StoreFile> candidates) throws IOException {
+ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, TableDescriptor htd) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompactSelection()");
+ LOG.debug("==> HBaseAtlasCoprocessor.postModifyTable()");
}
try {
activatePluginClassLoader();
- implRegionObserver.preCompactSelection(e, store, candidates);
+ implMasterObserver.postModifyTable(ctx, tableName, htd);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompactSelection()");
+ LOG.debug("<== HBaseAtlasCoprocessor.postModifyTable()");
}
}
@Override
- public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> c, HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCreateTable()");
+ LOG.debug("==> HBaseAtlasCoprocessor.postDeleteTable()");
}
try {
activatePluginClassLoader();
- implMasterObserver.preCreateTable(c, desc, regions);
+ implMasterObserver.postDeleteTable(ctx, tableName);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCreateTable()");
+ LOG.debug("<== HBaseAtlasCoprocessor.postDeleteTable()");
}
}
@Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete, WALEdit edit, Durability durability) throws IOException {
+ public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDelete()");
+ LOG.debug("==> HBaseAtlasCoprocessor.preCreateNamespace()");
}
try {
activatePluginClassLoader();
- implRegionObserver.preDelete(c, delete, edit, durability);
+ implMasterObserver.postCreateNamespace(ctx, ns);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDelete()");
+ LOG.debug("<== HBaseAtlasCoprocessor.preCreateNamespace()");
}
}
@Override
- public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, byte[] col) throws IOException {
+ public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, String ns) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteColumn()");
+ LOG.debug("==> HBaseAtlasCoprocessor.preDeleteNamespace()");
}
try {
activatePluginClassLoader();
- implMasterObserver.preDeleteColumn(c, tableName, col);
+ implMasterObserver.postDeleteNamespace(ctx, ns);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteColumn()");
+ LOG.debug("<== HBaseAtlasCoprocessor.preDeleteNamespace()");
}
}
-
@Override
- public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot) throws IOException {
+ public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteSnapshot()");
+ LOG.debug("==> HBaseAtlasCoprocessor.preModifyNamespace()");
}
try {
activatePluginClassLoader();
- implMasterObserver.preDeleteSnapshot(ctx, snapshot);
+ implMasterObserver.preModifyNamespace(ctx, ns);
} finally {
deactivatePluginClassLoader();
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteSnapshot()");
+ LOG.debug("<== HBaseAtlasCoprocessor.preModifyNamespace()");
}
}
@Override
- public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteTable()");
+ public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HBaseAtlasCoprocessor.postCloneSnapshot()");
}
try {
activatePluginClassLoader();
- implMasterObserver.preDeleteTable(c, tableName);
+ implMasterObserver.postCloneSnapshot(observerContext,snapshot,tableDescriptor);
} finally {
deactivatePluginClassLoader();
}
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteTable()");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()");
}
}
@Override
- public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDisableTable()");
+ public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");
}
try {
activatePluginClassLoader();
- implMasterObserver.preDisableTable(c, tableName);
+ implMasterObserver.postRestoreSnapshot(observerContext,snapshot,tableDescriptor);
} finally {
deactivatePluginClassLoader();
}
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDisableTable()");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()");
}
}
- @Override
- public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preEnableTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preEnableTable(c, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preEnableTable()");
+ private void activatePluginClassLoader() {
+ if(atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.activate();
}
}
- @Override
- public boolean preExists(ObserverContext<RegionCoprocessorEnvironment> c, Get get, boolean exists) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preExists()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preExists(c, get, exists);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preExists()");
+ private void deactivatePluginClassLoader() {
+ if(atlasPluginClassLoader != null) {
+ atlasPluginClassLoader.deactivate();
}
-
- return ret;
}
- @Override
- public void preFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preFlush()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preFlush(e);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preFlush()");
- }
- }
-
- @Override
- public void preGetClosestRowBefore(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, Result result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetClosestRowBefore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preGetClosestRowBefore(c, row, family, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetClosestRowBefore()");
- }
- }
-
- @Override
- public Result preIncrement(ObserverContext<RegionCoprocessorEnvironment> c, Increment increment) throws IOException {
- final Result ret;
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preIncrement()");
- }
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preIncrement(c, increment);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preIncrement()");
- }
-
- return ret;
- }
-
- @Override
- public long preIncrementColumnValue(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preIncrementColumnValue()");
- }
-
- final long ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preIncrementColumnValue(c, row, family, qualifier, amount, writeToWAL);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preIncrementColumnValue()");
- }
-
- return ret;
- }
-
- @Override
- public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, HColumnDescriptor descriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preModifyColumn()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preModifyColumn(c, tableName, descriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preModifyColumn()");
- }
- }
-
- @Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, HTableDescriptor htd) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preModifyTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preModifyTable(c, tableName, htd);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preModifyTable()");
- }
- }
-
- @Override
- public void preMove(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo region, ServerName srcServer, ServerName destServer) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preMove()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preMove(c, region, srcServer, destServer);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preMove()");
- }
- }
-
- @Override
- public void preOpen(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preOpen()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preOpen(e);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preOpen()");
- }
- }
-
- @Override
- public void preRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preRestoreSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preRestoreSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preRestoreSnapshot()");
- }
- }
-
- @Override
- public void preScannerClose(ObserverContext<RegionCoprocessorEnvironment> c, InternalScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preScannerClose()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preScannerClose(c, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preScannerClose()");
- }
- }
-
- @Override
- public boolean preScannerNext(ObserverContext<RegionCoprocessorEnvironment> c, InternalScanner s, List<Result> result, int limit, boolean hasNext) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preScannerNext()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preScannerNext(c, s, result, limit, hasNext);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preScannerNext()");
- }
-
- return ret;
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, RegionScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preScannerOpen()");
- }
-
- final RegionScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preScannerOpen(c, scan, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preShutdown()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preShutdown(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preShutdown()");
- }
- }
-
- @Override
- public void preSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSnapshot()");
- }
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preSplit(e);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSplit()");
- }
- }
-
- @Override
- public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preStopMaster()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preStopMaster(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preStopMaster()");
- }
- }
-
- @Override
- public void preStopRegionServer(ObserverContext<RegionServerCoprocessorEnvironment> env) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preStopRegionServer()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preStopRegionServer(env);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preStopRegionServer()");
- }
- }
-
- @Override
- public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo, boolean force) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preUnassign()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preUnassign(c, regionInfo, force);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preUnassign()");
- }
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSetUserQuota(ctx, userName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSetUserQuota()");
- }
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, TableName tableName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSetUserQuota(ctx, userName, tableName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSetUserQuota()");
- }
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, String namespace, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSetUserQuota(ctx, userName, namespace, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSetUserQuota()");
- }
- }
-
- @Override
- public void preSetTableQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSetTableQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSetTableQuota(ctx, tableName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSetTableQuota()");
- }
- }
-
- @Override
- public void preSetNamespaceQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSetNamespaceQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preSetNamespaceQuota(ctx, namespace, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSetNamespaceQuota()");
- }
- }
-
- @Override
- public void start(CoprocessorEnvironment env) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.start()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.start(env);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.start()");
- }
- }
-
- @Override
- public void prePut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit, Durability durability) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.prePut()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.prePut(c, put, edit, durability);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.prePut()");
- }
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> rEnv, Get get, List<Cell> result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetOp()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preGetOp(rEnv, get, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetOp()");
- }
- }
-
- @Override
- public void preRegionOffline(ObserverContext<MasterCoprocessorEnvironment> c, HRegionInfo regionInfo) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preRegionOffline()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preRegionOffline(c, regionInfo);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preRegionOffline()");
- }
- }
-
- @Override
- public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCreateNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preCreateNamespace(ctx, ns);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCreateNamespace()");
- }
- }
-
- @Override
- public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preDeleteNamespace(ctx, namespace);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteNamespace()");
- }
- }
-
- @Override
- public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preModifyNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preModifyNamespace(ctx, ns);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preModifyNamespace()");
- }
- }
-
- @Override
- public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<TableName> tableNamesList, List<HTableDescriptor> descriptors, String regex) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetTableDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postGetTableDescriptors(ctx, tableNamesList, descriptors, regex);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetTableDescriptors()");
- }
- }
-
- @Override
- public void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preMerge()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preMerge(ctx, regionA, regionB);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preMerge()");
- }
- }
-
- @Override
- public void prePrepareBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx, PrepareBulkLoadRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.prePrepareBulkLoad()");
- }
-
- try {
- activatePluginClassLoader();
- implBulkLoadObserver.prePrepareBulkLoad(ctx, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.prePrepareBulkLoad()");
- }
- }
-
- @Override
- public void preCleanupBulkLoad(ObserverContext<RegionCoprocessorEnvironment> ctx, CleanupBulkLoadRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCleanupBulkLoad()");
- }
-
- try {
- activatePluginClassLoader();
- implBulkLoadObserver.preCleanupBulkLoad(ctx, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCleanupBulkLoad()");
- }
- }
-
-
- @Override
- public void stop(CoprocessorEnvironment env) throws IOException {
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.stop()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.stop(env);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.stop()");
- }
- }
-
- @Override
- public void postMerge(ObserverContext<RegionServerCoprocessorEnvironment> c, Region regionA, Region regionB, Region mergedRegion) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postMerge()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.postMerge(c, regionA, regionB, mergedRegion);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postMerge()");
- }
- }
-
- @Override
- public void preMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB, List<Mutation> metaEntries) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preMergeCommit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preMergeCommit(ctx ,regionA, regionB, metaEntries);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preMergeCommit()");
- }
- }
-
- @Override
- public void postMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB, Region mergedRegion) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postMergeCommit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.postMergeCommit(ctx ,regionA, regionB, mergedRegion);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postMergeCommit()");
- }
- }
-
- @Override
- public void preRollBackMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preRollBackMerge()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preRollBackMerge(ctx, regionA, regionB);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preRollBackMerge()");
- }
- }
-
- @Override
- public void postRollBackMerge(ObserverContext<RegionServerCoprocessorEnvironment> ctx, Region regionA, Region regionB) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRollBackMerge()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.postRollBackMerge(ctx, regionA, regionB);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postRollBackMerge()");
- }
- }
-
- @Override
- public void preRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preRollWALWriterRequest()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preRollWALWriterRequest(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preRollWALWriterRequest()");
- }
- }
-
- @Override
- public void postRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRollWALWriterRequest()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.postRollWALWriterRequest(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postRollWALWriterRequest()");
- }
- }
-
- @Override
- public ReplicationEndpoint postCreateReplicationEndPoint(ObserverContext<RegionServerCoprocessorEnvironment> ctx, ReplicationEndpoint endpoint) {
-
- final ReplicationEndpoint ret;
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCreateReplicationEndPoint()");
- }
-
- try {
- activatePluginClassLoader();
- ret = implRegionServerObserver.postCreateReplicationEndPoint(ctx, endpoint);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCreateReplicationEndPoint()");
- }
-
- return ret;
- }
-
- @Override
- public void preReplicateLogEntries(ObserverContext<RegionServerCoprocessorEnvironment> ctx, List<WALEntry> entries, CellScanner cells) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preReplicateLogEntries()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.preReplicateLogEntries(ctx, entries, cells);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preReplicateLogEntries()");
- }
- }
-
- @Override
- public void postReplicateLogEntries(ObserverContext<RegionServerCoprocessorEnvironment> ctx, List<WALEntry> entries, CellScanner cells) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postReplicateLogEntries()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionServerObserver.postReplicateLogEntries(ctx, entries, cells);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postReplicateLogEntries()");
- }
- }
-
- @Override
- public void postOpen(ObserverContext<RegionCoprocessorEnvironment> c) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postOpen()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postOpen(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postOpen()");
- }
- }
-
- @Override
- public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> c) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postLogReplay()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postLogReplay(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postLogReplay()");
- }
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, KeyValueScanner memstoreScanner, InternalScanner s) throws IOException {
-
- final InternalScanner ret;
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preFlushScannerOpen()");
- }
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preFlushScannerOpen(c, store, memstoreScanner, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preFlushScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public InternalScanner preFlush(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner) throws IOException {
-
- final InternalScanner ret;
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preFlush()");
- }
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preFlush(c, store, scanner);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preFlush()");
- }
-
- return ret;
- }
-
- @Override
- public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postFlush()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postFlush(c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postFlush()");
- }
- }
-
- @Override
- public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postFlush()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postFlush(c, store, resultFile);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postFlush()");
- }
- }
-
- @Override
- public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<StoreFile> candidates, CompactionRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompactSelection()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preCompactSelection(c, store, candidates, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompactSelection()");
- }
- }
-
- @Override
- public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, ImmutableList<StoreFile> selected, CompactionRequest request) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCompactSelection()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCompactSelection(c, store, selected, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCompactSelection()");
- }
- }
-
- @Override
- public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, ImmutableList<StoreFile> selected) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCompactSelection()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCompactSelection(c, store, selected);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCompactSelection()");
- }
- }
-
- @Override
- public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner, ScanType scanType, CompactionRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompact()");
- }
-
- final InternalScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCompact(c, store, scanner, scanType, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompact()");
- }
-
- return ret;
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
- long earliestPutTs, InternalScanner s, CompactionRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompactScannerOpen()");
- }
-
- final InternalScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s,request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompactScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
- long earliestPutTs, InternalScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCompactScannerOpen()");
- }
-
- final InternalScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCompactScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionRequest request) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCompact()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCompact(c, store, resultFile, request);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCompact()");
- }
- }
-
- @Override
- public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCompact()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCompact(c, store, resultFile);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCompact()");
- }
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> c, byte[] splitRow) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preSplit(c, splitRow);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSplit()");
- }
- }
-
- @Override
- public void postSplit(ObserverContext<RegionCoprocessorEnvironment> c, Region l, Region r) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postSplit(c, l, r);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSplit()");
- }
- }
-
- @Override
- public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> ctx, byte[] splitKey, List<Mutation> metaEntries) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSplitBeforePONR()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preSplitBeforePONR(ctx, splitKey, metaEntries);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSplitBeforePONR()");
- }
- }
-
- @Override
- public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preSplitAfterPONR()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preSplitAfterPONR(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preSplitAfterPONR()");
- }
- }
-
- @Override
- public void preRollBackSplit(ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preRollBackSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preRollBackSplit(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preRollBackSplit()");
- }
- }
-
- @Override
- public void postRollBackSplit(ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRollBackSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postRollBackSplit(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postRollBackSplit()");
- }
- }
-
- @Override
- public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCompleteSplit()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCompleteSplit(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCompleteSplit()");
- }
- }
-
- @Override
- public void postClose(ObserverContext<RegionCoprocessorEnvironment> c, boolean abortRequested) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postClose()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postClose(c, abortRequested);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postClose()");
- }
- }
-
- @Override
- public void postGetClosestRowBefore(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, Result result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetClosestRowBefore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postGetClosestRowBefore(c, row, family, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetClosestRowBefore()");
- }
- }
-
- @Override
- public void postGetOp(ObserverContext<RegionCoprocessorEnvironment> c, Get get, List<Cell> result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetOp()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postGetOp(c, get, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetOp()");
- }
- }
-
- @Override
- public boolean postExists(ObserverContext<RegionCoprocessorEnvironment> c, Get get, boolean exists) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postExists()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postExists(c, get, exists);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postExists()");
- }
-
- return ret;
- }
-
- @Override
- public void postPut(ObserverContext<RegionCoprocessorEnvironment> c, Put put, WALEdit edit, Durability durability) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postPut()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postPut(c, put, edit, durability);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postPut()");
- }
- }
-
- @Override
- public void prePrepareTimeStampForDeleteVersion(ObserverContext<RegionCoprocessorEnvironment> c, Mutation mutation, Cell cell, byte[] byteNow, Get get) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.prePrepareTimeStampForDeleteVersion()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.prePrepareTimeStampForDeleteVersion(c, mutation, cell, byteNow, get);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.prePrepareTimeStampForDeleteVersion()");
- }
- }
-
- @Override
- public void postDelete(ObserverContext<RegionCoprocessorEnvironment> c, Delete delete, WALEdit edit, Durability durability) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDelete()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postDelete(c, delete, edit, durability);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDelete()");
- }
- }
-
- @Override
- public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c, MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preBatchMutate()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preBatchMutate(c, miniBatchOp);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preBatchMutate()");
- }
- }
-
- @Override
- public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c, MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postBatchMutate()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postBatchMutate(c, miniBatchOp);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postBatchMutate()");
- }
- }
-
- @Override
- public void postStartRegionOperation(ObserverContext<RegionCoprocessorEnvironment> ctx, Operation operation) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postStartRegionOperation()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postStartRegionOperation(ctx, operation);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postStartRegionOperation()");
- }
- }
-
- @Override
- public void postCloseRegionOperation(ObserverContext<RegionCoprocessorEnvironment> ctx, Operation operation) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCloseRegionOperation()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postCloseRegionOperation(ctx, operation);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCloseRegionOperation()");
- }
- }
-
- @Override
- public void postBatchMutateIndispensably(ObserverContext<RegionCoprocessorEnvironment> ctx, MiniBatchOperationInProgress<Mutation> miniBatchOp, boolean success) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postBatchMutateIndispensably()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postBatchMutateIndispensably(ctx, miniBatchOp, success);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postBatchMutateIndispensably()");
- }
- }
-
- @Override
- public boolean preCheckAndPutAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
- ByteArrayComparable comparator, Put put, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCheckAndPutAfterRowLock()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCheckAndPutAfterRowLock(c, row, family, qualifier, compareOp, comparator, put, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCheckAndPutAfterRowLock()");
- }
-
- return ret;
- }
-
- @Override
- public boolean postCheckAndPut(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
- ByteArrayComparable comparator, Put put, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCheckAndPut()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postCheckAndPut(c, row, family, qualifier, compareOp, comparator, put, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCheckAndPut()");
- }
-
- return ret;
- }
-
- @Override
- public boolean preCheckAndDeleteAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
- ByteArrayComparable comparator, Delete delete, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCheckAndDeleteAfterRowLock()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preCheckAndDeleteAfterRowLock(c, row, family, qualifier, compareOp, comparator, delete, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCheckAndDeleteAfterRowLock()");
- }
-
- return ret;
- }
-
- @Override
- public boolean postCheckAndDelete(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, CompareOp compareOp,
- ByteArrayComparable comparator, Delete delete, boolean result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCheckAndDelete()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postCheckAndDelete(c, row, family, qualifier, compareOp, comparator, delete, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCheckAndDelete()");
- }
-
- return ret;
- }
-
- @Override
- public long postIncrementColumnValue(ObserverContext<RegionCoprocessorEnvironment> c, byte[] row, byte[] family, byte[] qualifier, long amount, boolean writeToWAL, long result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postIncrementColumnValue()");
- }
-
- final long ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postIncrementColumnValue(c, row, family, qualifier, amount, writeToWAL, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postIncrementColumnValue()");
- }
-
- return ret;
- }
-
- @Override
- public Result preAppendAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> c, Append append) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preAppendAfterRowLock()");
- }
-
- final Result ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preAppendAfterRowLock(c, append);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preAppendAfterRowLock()");
- }
-
- return ret;
- }
-
- @Override
- public Result postAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append append, Result result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postAppend()");
- }
-
- final Result ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postAppend(c, append, result);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postAppend()");
- }
-
- return ret;
- }
-
- @Override
- public Result preIncrementAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> c, Increment increment) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preIncrementAfterRowLock()");
- }
-
- final Result ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preIncrementAfterRowLock(c, increment);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preIncrementAfterRowLock()");
- }
-
- return ret;
- }
-
- @Override
- public Result postIncrement(ObserverContext<RegionCoprocessorEnvironment> c, Increment increment, Result result) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postIncrement()");
- }
-
- final Result ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postIncrement(c, increment, result );
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postIncrement()");
- }
-
- return ret;
- }
-
- @Override
- public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, Scan scan, NavigableSet<byte[]> targetCols, KeyValueScanner s) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preStoreScannerOpen()");
- }
-
- final KeyValueScanner ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preStoreScannerOpen(c, store, scan, targetCols, s);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preStoreScannerOpen()");
- }
-
- return ret;
- }
-
- @Override
- public boolean postScannerNext(ObserverContext<RegionCoprocessorEnvironment> c, InternalScanner s, List<Result> result, int limit, boolean hasNext) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postScannerNext()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postScannerNext(c, s, result, limit, hasNext);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postScannerNext()");
- }
-
- return ret;
- }
-
- @Override
- public boolean postScannerFilterRow(ObserverContext<RegionCoprocessorEnvironment> c, InternalScanner s, byte[] currentRow, int offset, short length, boolean hasMore) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postScannerFilterRow()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postScannerFilterRow(c, s, currentRow, offset, length, hasMore);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postScannerFilterRow()");
- }
-
- return ret;
- }
-
- @Override
- public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preWALRestore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preWALRestore(ctx, info, logKey, logEdit);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preWALRestore()");
- }
- }
-
- @Override
- public void postWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> ctx, HRegionInfo info, WALKey logKey, WALEdit logEdit) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postWALRestore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postWALRestore(ctx, info, logKey, logEdit);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postWALRestore()");
- }
- }
-
- @Override
- public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx, List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postBulkLoadHFile()");
- }
-
- final boolean ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postBulkLoadHFile(ctx, familyPaths, hasLoaded);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postBulkLoadHFile()");
- }
-
- return ret;
- }
-
- @Override
- public Reader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx, FileSystem fs, Path p, FSDataInputStreamWrapper in, long size,
- CacheConfig cacheConf, Reference r, Reader reader) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preStoreFileReaderOpen()");
- }
-
- final Reader ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.preStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, reader);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preStoreFileReaderOpen()");
- }
-
- return ret;
- }
-
- @Override
- public Reader postStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> ctx, FileSystem fs, Path p, FSDataInputStreamWrapper in, long size,
- CacheConfig cacheConf, Reference r, Reader reader) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postStoreFileReaderOpen()");
- }
-
- final Reader ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postStoreFileReaderOpen(ctx, fs, p, in, size, cacheConf, r, reader);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postStoreFileReaderOpen()");
- }
-
- return ret;
- }
-
- @Override
- public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postMutationBeforeWAL()");
- }
-
- final Cell ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postMutationBeforeWAL(ctx, opType, mutation, oldCell, newCell);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postMutationBeforeWAL()");
- }
-
- return ret;
- }
-
- @Override
- public DeleteTracker postInstantiateDeleteTracker(ObserverContext<RegionCoprocessorEnvironment> ctx, DeleteTracker delTracker) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postInstantiateDeleteTracker()");
- }
-
- final DeleteTracker ret;
-
- try {
- activatePluginClassLoader();
- ret = implRegionObserver.postInstantiateDeleteTracker(ctx, delTracker);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postInstantiateDeleteTracker()");
- }
-
- return ret;
- }
-
- @Override
- public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCreateTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postCreateTable(ctx, desc, regions);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCreateTable()");
- }
- }
-
- @Override
- public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preCreateTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preCreateTableHandler(ctx, desc, regions);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preCreateTableHandler()");
- }
- }
-
- @Override
- public void postCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCreateTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postCreateTableHandler(ctx, desc, regions);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCreateTableHandler()");
- }
- }
-
- @Override
- public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteTable(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteTable()");
- }
- }
-
- @Override
- public void preDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preDeleteTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteTableHandler()");
- }
- }
-
- @Override
- public void postDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteTableHandler()");
- }
- }
-
- @Override
- public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preTruncateTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preTruncateTable(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preTruncateTable()");
- }
- }
-
- @Override
- public void postTruncateTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postTruncateTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postTruncateTable(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postTruncateTable()");
- }
- }
-
- @Override
- public void preTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preTruncateTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preTruncateTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preTruncateTableHandler()");
- }
- }
-
- @Override
- public void postTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postTruncateTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postTruncateTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postTruncateTableHandler()");
- }
- }
-
- @Override
- public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HTableDescriptor htd) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postModifyTable(ctx, tableName, htd);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyTable()");
- }
- }
-
- @Override
- public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HTableDescriptor htd) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preModifyTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preModifyTableHandler(ctx, tableName, htd);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preModifyTableHandler()");
- }
- }
-
- @Override
- public void postModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HTableDescriptor htd) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postModifyTableHandler(ctx, tableName, htd);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyTableHandler()");
- }
- }
-
- @Override
- public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor column) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postAddColumn()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postAddColumn(ctx, tableName, column);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postAddColumn()");
- }
- }
-
- @Override
- public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor column) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preAddColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preAddColumnHandler(ctx, tableName, column);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preAddColumnHandler()");
- }
- }
-
- @Override
- public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor column) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postAddColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postAddColumnHandler(ctx, tableName, column);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postAddColumnHandler()");
- }
- }
-
- @Override
- public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor descriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyColumn()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postModifyColumn(ctx, tableName, descriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyColumn()");
- }
- }
-
- @Override
- public void preModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor descriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preModifyColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preModifyColumnHandler(ctx, tableName, descriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preModifyColumnHandler()");
- }
- }
-
- @Override
- public void postModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, HColumnDescriptor descriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postModifyColumnHandler(ctx, tableName, descriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyColumnHandler()");
- }
- }
-
- @Override
- public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, byte[] c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteColumn()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteColumn(ctx, tableName, c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteColumn()");
- }
- }
-
- @Override
- public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, byte[] c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDeleteColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preDeleteColumnHandler(ctx, tableName, c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDeleteColumnHandler()");
- }
- }
-
- @Override
- public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, byte[] c) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteColumnHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteColumnHandler(ctx, tableName, c);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteColumnHandler()");
- }
- }
-
- @Override
- public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postEnableTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postEnableTable(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postEnableTable()");
- }
- }
-
- @Override
- public void preEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preEnableTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preEnableTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preEnableTableHandler()");
- }
- }
-
- @Override
- public void postEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postEnableTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postEnableTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postEnableTableHandler()");
- }
- }
-
- @Override
- public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDisableTable()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDisableTable(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDisableTable()");
- }
- }
-
- @Override
- public void preDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preDisableTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preDisableTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preDisableTableHandler()");
- }
- }
-
- @Override
- public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDisableTableHandler()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDisableTableHandler(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDisableTableHandler()");
- }
- }
-
- @Override
- public void postMove(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo region, ServerName srcServer, ServerName destServer) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postMove()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postMove(ctx, region, srcServer, destServer);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postMove()");
- }
- }
-
- @Override
- public void postAssign(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postAssign()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postAssign(ctx, regionInfo);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postAssign()");
- }
- }
-
- @Override
- public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo, boolean force) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postUnassign()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postUnassign(ctx, regionInfo, force);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postUnassign()");
- }
- }
-
- @Override
- public void postRegionOffline(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRegionOffline()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postRegionOffline(ctx, regionInfo);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postRegionOffline()");
- }
- }
-
- @Override
- public void postBalance(ObserverContext<MasterCoprocessorEnvironment> ctx, List<RegionPlan> plans) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postBalance()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postBalance(ctx, plans);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postBalance()");
- }
- }
-
- @Override
- public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> ctx, boolean oldValue, boolean newValue) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postBalanceSwitch()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postBalanceSwitch(ctx, oldValue, newValue);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postBalanceSwitch()");
- }
- }
-
- @Override
- public void preMasterInitialization(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preMasterInitialization()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preMasterInitialization(ctx);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preMasterInitialization()");
- }
- }
-
- @Override
- public void postSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSnapshot()");
- }
- }
-
- @Override
- public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preListSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preListSnapshot(ctx, snapshot);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preListSnapshot()");
- }
- }
-
- @Override
- public void postListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postListSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postListSnapshot(ctx, snapshot);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postListSnapshot()");
- }
- }
-
- @Override
- public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCloneSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postCloneSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()");
- }
- }
-
- @Override
- public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postRestoreSnapshot(ctx, snapshot, hTableDescriptor);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()");
- }
- }
-
- @Override
- public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx, SnapshotDescription snapshot) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteSnapshot()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteSnapshot(ctx, snapshot);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteSnapshot()");
- }
- }
-
- @Override
- public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<TableName> tableNamesList, List<HTableDescriptor> descriptors) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetTableDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preGetTableDescriptors(ctx, tableNamesList, descriptors);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetTableDescriptors()");
- }
- }
-
- @Override
- public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<HTableDescriptor> descriptors) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetTableDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postGetTableDescriptors(ctx, descriptors);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetTableDescriptors()");
- }
- }
-
- @Override
- public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<TableName> tableNamesList, List<HTableDescriptor> descriptors, String regex) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetTableDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preGetTableDescriptors(ctx, tableNamesList, descriptors, regex);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetTableDescriptors()");
- }
- }
-
- @Override
- public void preGetTableNames(ObserverContext<MasterCoprocessorEnvironment> ctx, List<HTableDescriptor> descriptors, String regex) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetTableNames()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preGetTableNames(ctx, descriptors, regex);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetTableNames()");
- }
- }
-
- @Override
- public void postGetTableNames(ObserverContext<MasterCoprocessorEnvironment> ctx, List<HTableDescriptor> descriptors, String regex) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetTableNames()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postGetTableNames(ctx, descriptors, regex);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetTableNames()");
- }
- }
-
- @Override
- public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCreateNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postCreateNamespace(ctx, ns);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postCreateNamespace()");
- }
- }
-
- @Override
- public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postDeleteNamespace(ctx, namespace);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteNamespace()");
- }
- }
-
- @Override
- public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyNamespace()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postModifyNamespace(ctx, ns);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyNamespace()");
- }
- }
-
- @Override
- public void preGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preGetNamespaceDescriptor()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preGetNamespaceDescriptor(ctx, namespace);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preGetNamespaceDescriptor()");
- }
- }
-
- @Override
- public void postGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> ctx, NamespaceDescriptor ns) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postGetNamespaceDescriptor()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postGetNamespaceDescriptor(ctx, ns);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postGetNamespaceDescriptor()");
- }
- }
-
- @Override
- public void preListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<NamespaceDescriptor> descriptors) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preListNamespaceDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preListNamespaceDescriptors(ctx, descriptors);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preListNamespaceDescriptors()");
- }
- }
-
- @Override
- public void postListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx, List<NamespaceDescriptor> descriptors) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postListNamespaceDescriptors()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postListNamespaceDescriptors(ctx, descriptors);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postListNamespaceDescriptors()");
- }
- }
-
- @Override
- public void preTableFlush(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preTableFlush()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.preTableFlush(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preTableFlush()");
- }
- }
-
- @Override
- public void postTableFlush(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postTableFlush()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postTableFlush(ctx, tableName);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postTableFlush()");
- }
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSetUserQuota(ctx, userName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSetUserQuota()");
- }
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, TableName tableName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSetUserQuota(ctx, userName, tableName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSetUserQuota()");
- }
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String userName, String namespace, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSetUserQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSetUserQuota(ctx, userName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSetUserQuota()");
- }
- }
-
- @Override
- public void postSetTableQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSetTableQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSetTableQuota(ctx, tableName, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSetTableQuota()");
- }
- }
-
- @Override
- public void postSetNamespaceQuota(ObserverContext<MasterCoprocessorEnvironment> ctx, String namespace, Quotas quotas) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postSetNamespaceQuota()");
- }
-
- try {
- activatePluginClassLoader();
- implMasterObserver.postSetNamespaceQuota(ctx, namespace, quotas);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postSetNamespaceQuota()");
- }
- }
-
- @Override
- public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> ctx, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.preWALRestore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.preWALRestore(ctx, info, logKey, logEdit);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.preWALRestore()");
- }
- }
-
- @Override
- public void postWALRestore(ObserverContext<RegionCoprocessorEnvironment> ctx, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException {
- if(LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postWALRestore()");
- }
-
- try {
- activatePluginClassLoader();
- implRegionObserver.postWALRestore(ctx, info, logKey, logEdit);
- } finally {
- deactivatePluginClassLoader();
- }
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postWALRestore()");
- }
- }
-
- private void activatePluginClassLoader() {
- if(atlasPluginClassLoader != null) {
- atlasPluginClassLoader.activate();
- }
- }
-
- private void deactivatePluginClassLoader() {
- if(atlasPluginClassLoader != null) {
- atlasPluginClassLoader.deactivate();
- }
- }
-
-
-
- // TODO : need override annotations for all of the following methods
- public void preMoveServers(final ObserverContext<MasterCoprocessorEnvironment> ctx, Set<HostAndPort> servers, String targetGroup) throws IOException {}
- public void postMoveServers(ObserverContext<MasterCoprocessorEnvironment> ctx, Set<HostAndPort> servers, String targetGroup) throws IOException {}
- public void preMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, Set<TableName> tables, String targetGroup) throws IOException {}
- public void postMoveTables(final ObserverContext<MasterCoprocessorEnvironment> ctx, Set<TableName> tables, String targetGroup) throws IOException {}
- public void preRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name) throws IOException {}
- public void postRemoveRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String name) throws IOException {}
- public void preBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {}
- public void postBalanceRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName, boolean balancerRan) throws IOException {}
- public void preAddRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String name) throws IOException {}
- public void postAddRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx, String name) throws IOException {}
}
diff --git a/addons/hbase-bridge/pom.xml b/addons/hbase-bridge/pom.xml
index 82f6010..a33bf30 100644
--- a/addons/hbase-bridge/pom.xml
+++ b/addons/hbase-bridge/pom.xml
@@ -31,8 +31,7 @@
<packaging>jar</packaging>
<properties>
- <hbase.version>1.2.1</hbase.version>
- <calcite.version>0.9.2-incubating</calcite.version>
+ <hadoop.version>3.0.3</hadoop.version>
</properties>
<dependencies>
@@ -51,21 +50,15 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
- <artifactId>atlas-client-v1</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.atlas</groupId>
- <artifactId>atlas-client-v2</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
</dependency>
@@ -92,11 +85,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
@@ -104,6 +99,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -166,6 +166,13 @@
</dependency>
<dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ <version>4.12</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
@@ -192,7 +199,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>12.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -213,10 +219,32 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
- <version>1.3.3</version>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-client-v2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <type>test-jar</type>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Intra-project dependencies -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
</dependency>
+
</dependencies>
<profiles>
@@ -247,11 +275,6 @@
</artifactItem>
<artifactItem>
<groupId>${project.groupId}</groupId>
- <artifactId>atlas-client-v1</artifactId>
- <version>${project.version}</version>
- </artifactItem>
- <artifactItem>
- <groupId>${project.groupId}</groupId>
<artifactId>atlas-client-common</artifactId>
<version>${project.version}</version>
</artifactItem>
@@ -296,11 +319,6 @@
<version>${jersey.version}</version>
</artifactItem>
<artifactItem>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </artifactItem>
- <artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
@@ -321,11 +339,6 @@
<version>${commons-conf.version}</version>
</artifactItem>
<artifactItem>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <version>${hbase.version}</version>
- </artifactItem>
- <artifactItem>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>${jersey.version}</version>
@@ -386,7 +399,6 @@
<webApp>
<contextPath>/</contextPath>
<descriptor>${project.basedir}/../../webapp/src/test/webapp/WEB-INF/web.xml</descriptor>
- <extraClasspath>${project.basedir}/../../webapp/target/test-classes/</extraClasspath>
</webApp>
<useTestScope>true</useTestScope>
<systemProperties>
@@ -428,6 +440,18 @@
<stopPort>31001</stopPort>
<stopWait>${jetty-maven-plugin.stopWait}</stopWait>
</configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.8</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>start-jetty</id>
@@ -502,7 +526,10 @@
<resources>
<resource>
<directory>${basedir}/../models</directory>
- <filtering>true</filtering>
+ <includes>
+ <include>0000-Area0/**</include>
+ <include>1000-Hadoop/**</include>
+ </includes>
</resource>
</resources>
</configuration>
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
index e7e9187..1825cd2 100644
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
+++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseAtlasHook.java
@@ -31,11 +31,12 @@ import org.apache.atlas.model.notification.HookNotification.EntityUpdateRequestV
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.ipc.RpcServer;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
@@ -45,6 +46,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -72,18 +74,22 @@ public class HBaseAtlasHook extends AtlasHook {
public static final String ATTR_TABLE_MAX_FILESIZE = "maxFileSize";
public static final String ATTR_TABLE_ISREADONLY = "isReadOnly";
public static final String ATTR_TABLE_ISCOMPACTION_ENABLED = "isCompactionEnabled";
+ public static final String ATTR_TABLE_ISNORMALIZATION_ENABLED = "isNormalizationEnabled";
public static final String ATTR_TABLE_REPLICATION_PER_REGION = "replicasPerRegion";
public static final String ATTR_TABLE_DURABLILITY = "durability";
+ public static final String ATTR_TABLE_NORMALIZATION_ENABLED = "isNormalizationEnabled";
// column family additional metadata
public static final String ATTR_CF_BLOOMFILTER_TYPE = "bloomFilterType";
public static final String ATTR_CF_COMPRESSION_TYPE = "compressionType";
public static final String ATTR_CF_COMPACTION_COMPRESSION_TYPE = "compactionCompressionType";
public static final String ATTR_CF_ENCRYPTION_TYPE = "encryptionType";
+ public static final String ATTR_CF_INMEMORY_COMPACTION_POLICY = "inMemoryCompactionPolicy";
public static final String ATTR_CF_KEEP_DELETE_CELLS = "keepDeletedCells";
public static final String ATTR_CF_MAX_VERSIONS = "maxVersions";
public static final String ATTR_CF_MIN_VERSIONS = "minVersions";
public static final String ATTR_CF_DATA_BLOCK_ENCODING = "dataBlockEncoding";
+ public static final String ATTR_CF_STORAGE_POLICY = "StoragePolicy";
public static final String ATTR_CF_TTL = "ttl";
public static final String ATTR_CF_BLOCK_CACHE_ENABLED = "blockCacheEnabled";
public static final String ATTR_CF_CACHED_BLOOM_ON_WRITE = "cacheBloomsOnWrite";
@@ -91,6 +97,9 @@ public class HBaseAtlasHook extends AtlasHook {
public static final String ATTR_CF_CACHED_INDEXES_ON_WRITE = "cacheIndexesOnWrite";
public static final String ATTR_CF_EVICT_BLOCK_ONCLOSE = "evictBlocksOnClose";
public static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN = "prefetchBlocksOnOpen";
+ public static final String ATTR_CF_NEW_VERSION_BEHAVIOR = "newVersionBehavior";
+ public static final String ATTR_CF_MOB_ENABLED = "isMobEnabled";
+ public static final String ATTR_CF_MOB_COMPATCTPARTITION_POLICY = "mobCompactPartitionPolicy";
public static final String HBASE_NAMESPACE_QUALIFIED_NAME = "%s@%s";
public static final String HBASE_TABLE_QUALIFIED_NAME_FORMAT = "%s:%s@%s";
@@ -153,7 +162,7 @@ public class HBaseAtlasHook extends AtlasHook {
public void createAtlasInstances(HBaseOperationContext hbaseOperationContext) {
- HBaseAtlasHook.OPERATION operation = hbaseOperationContext.getOperation();
+ OPERATION operation = hbaseOperationContext.getOperation();
LOG.info("HBaseAtlasHook(operation={})", operation);
@@ -396,13 +405,15 @@ public class HBaseAtlasHook extends AtlasHook {
table.setAttribute(ATTR_PARAMETERS, hbaseOperationContext.getHbaseConf());
table.setAttribute(ATTR_NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpace));
- HTableDescriptor htableDescriptor = hbaseOperationContext.gethTableDescriptor();
- if (htableDescriptor != null) {
- table.setAttribute(ATTR_TABLE_MAX_FILESIZE, htableDescriptor.getMaxFileSize());
- table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htableDescriptor.getRegionReplication());
- table.setAttribute(ATTR_TABLE_ISREADONLY, htableDescriptor.isReadOnly());
- table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htableDescriptor.isCompactionEnabled());
- table.setAttribute(ATTR_TABLE_DURABLILITY, (htableDescriptor.getDurability() != null ? htableDescriptor.getDurability().name() : null));
+ TableDescriptor tableDescriptor = hbaseOperationContext.gethTableDescriptor();
+ if (tableDescriptor != null) {
+ table.setAttribute(ATTR_TABLE_MAX_FILESIZE, tableDescriptor.getMaxFileSize());
+ table.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, tableDescriptor.getRegionReplication());
+ table.setAttribute(ATTR_TABLE_ISREADONLY, tableDescriptor.isReadOnly());
+ table.setAttribute(ATTR_TABLE_ISNORMALIZATION_ENABLED, tableDescriptor.isNormalizationEnabled());
+ table.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, tableDescriptor.isCompactionEnabled());
+ table.setAttribute(ATTR_TABLE_DURABLILITY, (tableDescriptor.getDurability() != null ? tableDescriptor.getDurability().name() : null));
+ table.setAttribute(ATTR_TABLE_NORMALIZATION_ENABLED, tableDescriptor.isNormalizationEnabled());
}
switch (operation) {
@@ -426,11 +437,11 @@ public class HBaseAtlasHook extends AtlasHook {
private List<AtlasEntity> buildColumnFamilies(HBaseOperationContext hbaseOperationContext, AtlasEntity nameSpace, AtlasEntity table) {
List<AtlasEntity> columnFamilies = new ArrayList<>();
- HColumnDescriptor[] hColumnDescriptors = hbaseOperationContext.gethColumnDescriptors();
+ ColumnFamilyDescriptor[] columnFamilyDescriptors = hbaseOperationContext.gethColumnDescriptors();
- if (hColumnDescriptors != null) {
- for (HColumnDescriptor hColumnDescriptor : hColumnDescriptors) {
- AtlasEntity columnFamily = buildColumnFamily(hbaseOperationContext, hColumnDescriptor, nameSpace, table);
+ if (columnFamilyDescriptors != null) {
+ for (ColumnFamilyDescriptor columnFamilyDescriptor : columnFamilyDescriptors) {
+ AtlasEntity columnFamily = buildColumnFamily(hbaseOperationContext, columnFamilyDescriptor, nameSpace, table);
columnFamilies.add(columnFamily);
}
@@ -439,9 +450,9 @@ public class HBaseAtlasHook extends AtlasHook {
return columnFamilies;
}
- private AtlasEntity buildColumnFamily(HBaseOperationContext hbaseOperationContext, HColumnDescriptor hColumnDescriptor, AtlasEntity nameSpace, AtlasEntity table) {
+ private AtlasEntity buildColumnFamily(HBaseOperationContext hbaseOperationContext, ColumnFamilyDescriptor columnFamilyDescriptor, AtlasEntity nameSpace, AtlasEntity table) {
AtlasEntity columnFamily = new AtlasEntity(HBaseDataTypes.HBASE_COLUMN_FAMILY.getName());
- String columnFamilyName = hColumnDescriptor.getNameAsString();
+ String columnFamilyName = columnFamilyDescriptor.getNameAsString();
String tableName = (String) table.getAttribute(ATTR_NAME);
String nameSpaceName = (String) nameSpace.getAttribute(ATTR_NAME);
String columnFamilyQName = getColumnFamilyQualifiedName(clusterName, nameSpaceName, tableName, columnFamilyName);
@@ -453,22 +464,27 @@ public class HBaseAtlasHook extends AtlasHook {
columnFamily.setAttribute(ATTR_OWNER, hbaseOperationContext.getOwner());
columnFamily.setAttribute(ATTR_TABLE, AtlasTypeUtil.getAtlasObjectId(table));
- if (hColumnDescriptor!= null) {
- columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, hColumnDescriptor.isBlockCacheEnabled());
- columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (hColumnDescriptor.getBloomFilterType() != null ? hColumnDescriptor.getBloomFilterType().name():null));
- columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, hColumnDescriptor.isCacheBloomsOnWrite());
- columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, hColumnDescriptor.isCacheDataOnWrite());
- columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, hColumnDescriptor.isCacheIndexesOnWrite());
- columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (hColumnDescriptor.getCompactionCompressionType() != null ? hColumnDescriptor.getCompactionCompressionType().name():null));
- columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, (hColumnDescriptor.getCompressionType() != null ? hColumnDescriptor.getCompressionType().name():null));
- columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (hColumnDescriptor.getDataBlockEncoding() != null ? hColumnDescriptor.getDataBlockEncoding().name():null));
- columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, hColumnDescriptor.getEncryptionType());
- columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, hColumnDescriptor.isEvictBlocksOnClose());
- columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( hColumnDescriptor.getKeepDeletedCells() != null ? hColumnDescriptor.getKeepDeletedCells().name():null));
- columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, hColumnDescriptor.getMaxVersions());
- columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, hColumnDescriptor.getMinVersions());
- columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, hColumnDescriptor.isPrefetchBlocksOnOpen());
- columnFamily.setAttribute(ATTR_CF_TTL, hColumnDescriptor.getTimeToLive());
+ if (columnFamilyDescriptor!= null) {
+ columnFamily.setAttribute(ATTR_CF_BLOCK_CACHE_ENABLED, columnFamilyDescriptor.isBlockCacheEnabled());
+ columnFamily.setAttribute(ATTR_CF_BLOOMFILTER_TYPE, (columnFamilyDescriptor.getBloomFilterType() != null ? columnFamilyDescriptor.getBloomFilterType().name():null));
+ columnFamily.setAttribute(ATTR_CF_CACHED_BLOOM_ON_WRITE, columnFamilyDescriptor.isCacheBloomsOnWrite());
+ columnFamily.setAttribute(ATTR_CF_CACHED_DATA_ON_WRITE, columnFamilyDescriptor.isCacheDataOnWrite());
+ columnFamily.setAttribute(ATTR_CF_CACHED_INDEXES_ON_WRITE, columnFamilyDescriptor.isCacheIndexesOnWrite());
+ columnFamily.setAttribute(ATTR_CF_COMPACTION_COMPRESSION_TYPE, (columnFamilyDescriptor.getCompactionCompressionType() != null ? columnFamilyDescriptor.getCompactionCompressionType().name():null));
+ columnFamily.setAttribute(ATTR_CF_COMPRESSION_TYPE, (columnFamilyDescriptor.getCompressionType() != null ? columnFamilyDescriptor.getCompressionType().name():null));
+ columnFamily.setAttribute(ATTR_CF_DATA_BLOCK_ENCODING, (columnFamilyDescriptor.getDataBlockEncoding() != null ? columnFamilyDescriptor.getDataBlockEncoding().name():null));
+ columnFamily.setAttribute(ATTR_CF_ENCRYPTION_TYPE, columnFamilyDescriptor.getEncryptionType());
+ columnFamily.setAttribute(ATTR_CF_EVICT_BLOCK_ONCLOSE, columnFamilyDescriptor.isEvictBlocksOnClose());
+ columnFamily.setAttribute(ATTR_CF_INMEMORY_COMPACTION_POLICY, (columnFamilyDescriptor.getInMemoryCompaction() != null ? columnFamilyDescriptor.getInMemoryCompaction().name():null));
+ columnFamily.setAttribute(ATTR_CF_KEEP_DELETE_CELLS, ( columnFamilyDescriptor.getKeepDeletedCells() != null ? columnFamilyDescriptor.getKeepDeletedCells().name():null));
+ columnFamily.setAttribute(ATTR_CF_MAX_VERSIONS, columnFamilyDescriptor.getMaxVersions());
+ columnFamily.setAttribute(ATTR_CF_MIN_VERSIONS, columnFamilyDescriptor.getMinVersions());
+ columnFamily.setAttribute(ATTR_CF_NEW_VERSION_BEHAVIOR, columnFamilyDescriptor.isNewVersionBehavior());
+ columnFamily.setAttribute(ATTR_CF_MOB_ENABLED, columnFamilyDescriptor.isMobEnabled());
+ columnFamily.setAttribute(ATTR_CF_MOB_COMPATCTPARTITION_POLICY, ( columnFamilyDescriptor.getMobCompactPartitionPolicy() != null ? columnFamilyDescriptor.getMobCompactPartitionPolicy().name():null));
+ columnFamily.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, columnFamilyDescriptor.isPrefetchBlocksOnOpen());
+ columnFamily.setAttribute(ATTR_CF_STORAGE_POLICY, columnFamilyDescriptor.getStoragePolicy());
+ columnFamily.setAttribute(ATTR_CF_TTL, columnFamilyDescriptor.getTimeToLive());
}
switch (hbaseOperationContext.getOperation()) {
@@ -497,21 +513,24 @@ public class HBaseAtlasHook extends AtlasHook {
if (tableName != null) {
ret = tableName.getNameAsString();
} else {
- HTableDescriptor tableDescriptor = hbaseOperationContext.gethTableDescriptor();
+ TableDescriptor tableDescriptor = hbaseOperationContext.gethTableDescriptor();
- ret = (tableDescriptor != null) ? tableDescriptor.getNameAsString() : null;
+ ret = (tableDescriptor != null) ? tableDescriptor.getTableName().getNameAsString() : null;
}
return ret;
}
- public void sendHBaseNameSpaceOperation(final NamespaceDescriptor namespaceDescriptor, final String nameSpace, final OPERATION operation) {
+ public void sendHBaseNameSpaceOperation(final NamespaceDescriptor namespaceDescriptor, final String nameSpace, final OPERATION operation, ObserverContext<MasterCoprocessorEnvironment> ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasHook.sendHBaseNameSpaceOperation()");
}
try {
- HBaseOperationContext hbaseOperationContext = handleHBaseNameSpaceOperation(namespaceDescriptor, nameSpace, operation);
+ final UserGroupInformation ugi = getUGI(ctx);
+ final User user = getActiveUser(ctx);
+ final String userName = (user != null) ? user.getShortName() : null;
+ HBaseOperationContext hbaseOperationContext = handleHBaseNameSpaceOperation(namespaceDescriptor, nameSpace, operation, ugi, userName);
sendNotification(hbaseOperationContext);
} catch (Throwable t) {
@@ -523,13 +542,16 @@ public class HBaseAtlasHook extends AtlasHook {
}
}
- public void sendHBaseTableOperation(final HTableDescriptor hTableDescriptor, final TableName tableName, final OPERATION operation) {
+ public void sendHBaseTableOperation(TableDescriptor tableDescriptor, final TableName tableName, final OPERATION operation, ObserverContext<MasterCoprocessorEnvironment> ctx) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasHook.sendHBaseTableOperation()");
}
try {
- HBaseOperationContext hbaseOperationContext = handleHBaseTableOperation(hTableDescriptor, tableName, operation);
+ final UserGroupInformation ugi = getUGI(ctx);
+ final User user = getActiveUser(ctx);
+ final String userName = (user != null) ? user.getShortName() : null;
+ HBaseOperationContext hbaseOperationContext = handleHBaseTableOperation(tableDescriptor, tableName, operation, ugi, userName);
sendNotification(hbaseOperationContext);
} catch (Throwable t) {
@@ -541,24 +563,6 @@ public class HBaseAtlasHook extends AtlasHook {
}
}
- public void sendHBaseColumnFamilyOperation(final HColumnDescriptor hColumnDescriptor, final TableName tableName, final String columnFamily, final OPERATION operation) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasHook.sendHBaseColumnFamilyOperation()");
- }
-
- try {
- HBaseOperationContext hbaseOperationContext = handleHBaseColumnFamilyOperation(hColumnDescriptor, tableName, columnFamily, operation);
-
- sendNotification(hbaseOperationContext);
- } catch (Throwable t) {
- LOG.error("<== HBaseAtlasHook.sendHBaseColumnFamilyOperation(): failed to send notification", t);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasHook.sendHBaseColumnFamilyOperation()");
- }
- }
-
private void sendNotification(HBaseOperationContext hbaseOperationContext) {
UserGroupInformation ugi = hbaseOperationContext.getUgi();
@@ -569,15 +573,11 @@ public class HBaseAtlasHook extends AtlasHook {
notifyEntities(hbaseOperationContext.getMessages(), ugi);
}
- private HBaseOperationContext handleHBaseNameSpaceOperation(NamespaceDescriptor namespaceDescriptor, String nameSpace, OPERATION operation) {
+ private HBaseOperationContext handleHBaseNameSpaceOperation(NamespaceDescriptor namespaceDescriptor, String nameSpace, OPERATION operation, UserGroupInformation ugi, String userName) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasHook.handleHBaseNameSpaceOperation()");
}
- UserGroupInformation ugi = getUGI();
- User user = getActiveUser();
- String userName = (user != null) ? user.getShortName() : null;
-
HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(namespaceDescriptor, nameSpace, operation, ugi, userName, userName);
createAtlasInstances(hbaseOperationContext);
@@ -588,24 +588,21 @@ public class HBaseAtlasHook extends AtlasHook {
return hbaseOperationContext;
}
- private HBaseOperationContext handleHBaseTableOperation(HTableDescriptor hTableDescriptor, TableName tableName, OPERATION operation) {
+ private HBaseOperationContext handleHBaseTableOperation(TableDescriptor tableDescriptor, TableName tableName, OPERATION operation, UserGroupInformation ugi, String userName) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasHook.handleHBaseTableOperation()");
}
- UserGroupInformation ugi = getUGI();
- User user = getActiveUser();
- String userName = (user != null) ? user.getShortName() : null;
Map<String, String> hbaseConf = null;
String owner = null;
String tableNameSpace = null;
TableName hbaseTableName = null;
- HColumnDescriptor[] hColumnDescriptors = null;
+ ColumnFamilyDescriptor[] columnFamilyDescriptors = null;
- if (hTableDescriptor != null) {
- owner = hTableDescriptor.getOwnerString();
- hbaseConf = hTableDescriptor.getConfiguration();
- hbaseTableName = hTableDescriptor.getTableName();
+ if (tableDescriptor != null) {
+ owner = tableDescriptor.getOwnerString();
+ hbaseConf = null;
+ hbaseTableName = tableDescriptor.getTableName();
if (hbaseTableName != null) {
tableNameSpace = hbaseTableName.getNamespaceAsString();
if (tableNameSpace == null) {
@@ -618,11 +615,11 @@ public class HBaseAtlasHook extends AtlasHook {
owner = userName;
}
- if (hTableDescriptor != null) {
- hColumnDescriptors = hTableDescriptor.getColumnFamilies();
+ if (tableDescriptor != null) {
+ columnFamilyDescriptors = tableDescriptor.getColumnFamilies();
}
- HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(tableNameSpace, hTableDescriptor, tableName, hColumnDescriptors, operation, ugi, userName, owner, hbaseConf);
+ HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(tableNameSpace, tableDescriptor, tableName, columnFamilyDescriptors, operation, ugi, userName, owner, hbaseConf);
createAtlasInstances(hbaseOperationContext);
if (LOG.isDebugEnabled()) {
@@ -631,27 +628,24 @@ public class HBaseAtlasHook extends AtlasHook {
return hbaseOperationContext;
}
- private HBaseOperationContext handleHBaseColumnFamilyOperation(HColumnDescriptor hColumnDescriptor, TableName tableName, String columnFamily, OPERATION operation) {
+ private HBaseOperationContext handleHBaseColumnFamilyOperation(ColumnFamilyDescriptor columnFamilyDescriptor, TableName tableName, String columnFamily, OPERATION operation, UserGroupInformation ugi, String userName) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasHook.handleHBaseColumnFamilyOperation()");
}
- UserGroupInformation ugi = getUGI();
- User user = getActiveUser();
- String userName = (user != null) ? user.getShortName() : null;
String owner = userName;
- Map<String, String> hbaseConf = null;
+ Map<String, String> hbaseConf = new HashMap<>();
String tableNameSpace = tableName.getNamespaceAsString();
if (tableNameSpace == null) {
tableNameSpace = tableName.getNameWithNamespaceInclAsString();
}
- if (hColumnDescriptor != null) {
- hbaseConf = hColumnDescriptor.getConfiguration();
+ if (columnFamilyDescriptor != null) {
+ hbaseConf = columnFamilyDescriptor.getConfiguration();
}
- HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(tableNameSpace, tableName, hColumnDescriptor, columnFamily, operation, ugi, userName, owner, hbaseConf);
+ HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(tableNameSpace, tableName, columnFamilyDescriptor, columnFamily, operation, ugi, userName, owner, hbaseConf);
createAtlasInstances(hbaseOperationContext);
if (LOG.isDebugEnabled()) {
@@ -660,26 +654,12 @@ public class HBaseAtlasHook extends AtlasHook {
return hbaseOperationContext;
}
- private User getActiveUser() {
- User user = RpcServer.getRequestUser();
- if (user == null) {
- // for non-rpc handling, fallback to system user
- try {
- user = User.getCurrent();
- } catch (IOException e) {
- LOG.error("Unable to find the current user");
- user = null;
- }
- }
- return user;
- }
-
- private UserGroupInformation getUGI() {
+ private UserGroupInformation getUGI(ObserverContext<?> ctx) {
UserGroupInformation ugi = null;
- User user = getActiveUser();
-
+ User user = null;
try {
- ugi = UserGroupInformation.getLoginUser();
+ user = getActiveUser(ctx);
+ ugi = UserGroupInformation.getLoginUser();
} catch (Exception e) {
// not setting the UGI here
}
@@ -693,4 +673,8 @@ public class HBaseAtlasHook extends AtlasHook {
LOG.info("HBaseAtlasHook: UGI: {}", ugi);
return ugi;
}
+
+ private User getActiveUser(ObserverContext<?> ctx) throws IOException {
+ return (User)ctx.getCaller().orElse(User.getCurrent());
+ }
}
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseBridge.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseBridge.java
index 8372f02..17d617d 100644
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseBridge.java
+++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/bridge/HBaseBridge.java
@@ -39,10 +39,14 @@ import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,6 +88,7 @@ public class HBaseBridge {
private static final String ATTR_TABLE_ISCOMPACTION_ENABLED = "isCompactionEnabled";
private static final String ATTR_TABLE_REPLICATION_PER_REGION = "replicasPerRegion";
private static final String ATTR_TABLE_DURABLILITY = "durability";
+ private static final String ATTR_TABLE_NORMALIZATION_ENABLED = "isNormalizationEnabled";
// column family metadata
private static final String ATTR_CF_BLOOMFILTER_TYPE = "bloomFilterType";
@@ -102,6 +107,10 @@ public class HBaseBridge {
private static final String ATTR_CF_EVICT_BLOCK_ONCLOSE = "evictBlocksOnClose";
private static final String ATTR_CF_PREFETCH_BLOCK_ONOPEN = "prefetchBlocksOnOpen";
private static final String ATTRIBUTE_QUALIFIED_NAME = "qualifiedName";
+ private static final String ATTR_CF_INMEMORY_COMPACTION_POLICY = "inMemoryCompactionPolicy";
+ private static final String ATTR_CF_MOB_COMPATCTPARTITION_POLICY = "mobCompactPartitionPolicy";
+ private static final String ATTR_CF_MOB_ENABLED = "isMobEnabled";
+ private static final String ATTR_CF_NEW_VERSION_BEHAVIOR = "newVersionBehavior";
private static final String HBASE_NAMESPACE_QUALIFIED_NAME = "%s@%s";
private static final String HBASE_TABLE_QUALIFIED_NAME_FORMAT = "%s:%s@%s";
@@ -109,7 +118,7 @@ public class HBaseBridge {
private final String clusterName;
private final AtlasClientV2 atlasClientV2;
- private final HBaseAdmin hbaseAdmin;
+ private final Admin hbaseAdmin;
public static void main(String[] args) {
@@ -199,11 +208,13 @@ public class HBaseBridge {
LOG.info("checking HBase availability..");
- HBaseAdmin.checkHBaseAvailable(conf);
+ HBaseAdmin.available(conf);
LOG.info("HBase is available");
- hbaseAdmin = new HBaseAdmin(conf);
+ Connection conn = ConnectionFactory.createConnection(conf);
+
+ hbaseAdmin = conn.getAdmin();
}
private boolean importHBaseEntities(String namespaceToImport, String tableToImport) throws Exception {
@@ -238,11 +249,11 @@ public class HBaseBridge {
}
public void importTable(final String tableName) throws Exception {
- String tableNameStr = null;
- HTableDescriptor[] htds = hbaseAdmin.listTables(Pattern.compile(tableName));
+ String tableNameStr = null;
+ TableDescriptor[] htds = hbaseAdmin.listTables(Pattern.compile(tableName));
if (ArrayUtils.isNotEmpty(htds)) {
- for (HTableDescriptor htd : htds) {
+ for (TableDescriptor htd : htds) {
String tblNameWithNameSpace = htd.getTableName().getNameWithNamespaceInclAsString();
String tblNameWithOutNameSpace = htd.getTableName().getNameAsString();
@@ -263,7 +274,7 @@ public class HBaseBridge {
String nsName = new String(nsByte);
NamespaceDescriptor nsDescriptor = hbaseAdmin.getNamespaceDescriptor(nsName);
AtlasEntityWithExtInfo entity = createOrUpdateNameSpace(nsDescriptor);
- HColumnDescriptor[] hcdts = htd.getColumnFamilies();
+ ColumnFamilyDescriptor[] hcdts = htd.getColumnFamilies();
createOrUpdateTable(nsName, tableNameStr, entity.getEntity(), htd, hcdts);
}
@@ -283,11 +294,11 @@ public class HBaseBridge {
}
}
- HTableDescriptor[] htds = hbaseAdmin.listTables();
+ TableDescriptor[] htds = hbaseAdmin.listTables();
if (ArrayUtils.isNotEmpty(htds)) {
- for (HTableDescriptor htd : htds) {
- String tableName = htd.getNameAsString();
+ for (TableDescriptor htd : htds) {
+ String tableName = htd.getTableName().getNameAsString();
importTable(tableName);
}
@@ -297,7 +308,7 @@ public class HBaseBridge {
private void importNameSpaceWithTable(String namespaceToImport, String tableToImport) throws Exception {
importNameSpace(namespaceToImport);
- List<HTableDescriptor> hTableDescriptors = new ArrayList<>();
+ List<TableDescriptor> hTableDescriptors = new ArrayList<>();
if (StringUtils.isEmpty(tableToImport)) {
List<NamespaceDescriptor> matchingNameSpaceDescriptors = getMatchingNameSpaces(namespaceToImport);
@@ -308,13 +319,13 @@ public class HBaseBridge {
} else {
tableToImport = namespaceToImport +":" + tableToImport;
- HTableDescriptor[] htds = hbaseAdmin.listTables(Pattern.compile(tableToImport));
+ TableDescriptor[] htds = hbaseAdmin.listTables(Pattern.compile(tableToImport));
hTableDescriptors.addAll(Arrays.asList(htds));
}
if (CollectionUtils.isNotEmpty(hTableDescriptors)) {
- for (HTableDescriptor htd : hTableDescriptors) {
+ for (TableDescriptor htd : hTableDescriptors) {
String tblName = htd.getTableName().getNameAsString();
importTable(tblName);
@@ -339,11 +350,11 @@ public class HBaseBridge {
return ret;
}
- private List<HTableDescriptor> getTableDescriptors(List<NamespaceDescriptor> namespaceDescriptors) throws Exception {
- List<HTableDescriptor> ret = new ArrayList<>();
+ private List<TableDescriptor> getTableDescriptors(List<NamespaceDescriptor> namespaceDescriptors) throws Exception {
+ List<TableDescriptor> ret = new ArrayList<>();
for(NamespaceDescriptor namespaceDescriptor:namespaceDescriptors) {
- HTableDescriptor[] tableDescriptors = hbaseAdmin.listTableDescriptorsByNamespace(namespaceDescriptor.getName());
+ TableDescriptor[] tableDescriptors = hbaseAdmin.listTableDescriptorsByNamespace(namespaceDescriptor.getName());
ret.addAll(Arrays.asList(tableDescriptors));
}
@@ -374,7 +385,7 @@ public class HBaseBridge {
return nsEntity;
}
- protected AtlasEntityWithExtInfo createOrUpdateTable(String nameSpace, String tableName, AtlasEntity nameSapceEntity, HTableDescriptor htd, HColumnDescriptor[] hcdts) throws Exception {
+ protected AtlasEntityWithExtInfo createOrUpdateTable(String nameSpace, String tableName, AtlasEntity nameSapceEntity, TableDescriptor htd, ColumnFamilyDescriptor[] hcdts) throws Exception {
String owner = htd.getOwnerString();
String tblQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName);
AtlasEntityWithExtInfo ret = findTableEntityInAtlas(tblQualifiedName);
@@ -414,13 +425,13 @@ public class HBaseBridge {
return ret;
}
- protected List<AtlasEntityWithExtInfo> createOrUpdateColumnFamilies(String nameSpace, String tableName, String owner, HColumnDescriptor[] hcdts , AtlasEntity tableEntity) throws Exception {
+ protected List<AtlasEntityWithExtInfo> createOrUpdateColumnFamilies(String nameSpace, String tableName, String owner, ColumnFamilyDescriptor[] hcdts , AtlasEntity tableEntity) throws Exception {
List<AtlasEntityWithExtInfo > ret = new ArrayList<>();
if (hcdts != null) {
AtlasObjectId tableId = AtlasTypeUtil.getAtlasObjectId(tableEntity);
- for (HColumnDescriptor columnFamilyDescriptor : hcdts) {
+ for (ColumnFamilyDescriptor columnFamilyDescriptor : hcdts) {
String cfName = columnFamilyDescriptor.getNameAsString();
String cfQualifiedName = getColumnFamilyQualifiedName(clusterName, nameSpace, tableName, cfName);
AtlasEntityWithExtInfo cfEntity = findColumnFamiltyEntityInAtlas(cfQualifiedName);
@@ -512,7 +523,7 @@ public class HBaseBridge {
return ret;
}
- private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, HTableDescriptor htd, AtlasEntity atlasEntity) {
+ private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, TableDescriptor htd, AtlasEntity atlasEntity) {
AtlasEntity ret = null;
if (atlasEntity == null) {
@@ -535,11 +546,12 @@ public class HBaseBridge {
ret.setAttribute(ATTR_TABLE_ISREADONLY, htd.isReadOnly());
ret.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htd.isCompactionEnabled());
ret.setAttribute(ATTR_TABLE_DURABLILITY, (htd.getDurability() != null ? htd.getDurability().name() : null));
+ ret.setAttribute(ATTR_TABLE_NORMALIZATION_ENABLED, htd.isNormalizationEnabled());
return ret;
}
- private AtlasEntity getColumnFamilyEntity(String nameSpace, String tableName, String owner, HColumnDescriptor hcdt, AtlasObjectId tableId, AtlasEntity atlasEntity){
+ private AtlasEntity getColumnFamilyEntity(String nameSpace, String tableName, String owner, ColumnFamilyDescriptor hcdt, AtlasObjectId tableId, AtlasEntity atlasEntity){
AtlasEntity ret = null;
if (atlasEntity == null) {
@@ -572,6 +584,10 @@ public class HBaseBridge {
ret.setAttribute(ATTR_CF_MIN_VERSIONS, hcdt.getMinVersions());
ret.setAttribute(ATTR_CF_PREFETCH_BLOCK_ONOPEN, hcdt.isPrefetchBlocksOnOpen());
ret.setAttribute(ATTR_CF_TTL, hcdt.getTimeToLive());
+ ret.setAttribute(ATTR_CF_INMEMORY_COMPACTION_POLICY, (hcdt.getInMemoryCompaction() != null ? hcdt.getInMemoryCompaction().name():null));
+ ret.setAttribute(ATTR_CF_MOB_COMPATCTPARTITION_POLICY, ( hcdt.getMobCompactPartitionPolicy() != null ? hcdt.getMobCompactPartitionPolicy().name():null));
+ ret.setAttribute(ATTR_CF_MOB_ENABLED,hcdt.isMobEnabled());
+ ret.setAttribute(ATTR_CF_NEW_VERSION_BEHAVIOR,hcdt.isNewVersionBehavior());
return ret;
}
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
index af8afd4..313132d 100644
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
+++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessor.java
@@ -20,21 +20,24 @@ package org.apache.atlas.hbase.hook;
import org.apache.atlas.hbase.bridge.HBaseAtlasHook;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.SnapshotDescription;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor;
+import org.apache.hadoop.hbase.coprocessor.BulkLoadObserver;
import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.coprocessor.RegionServerObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-public class HBaseAtlasCoprocessor extends HBaseAtlasCoprocessorBase {
+public class HBaseAtlasCoprocessor implements MasterCoprocessor, MasterObserver, RegionObserver, RegionServerObserver {
private static final Logger LOG = LoggerFactory.getLogger(HBaseAtlasCoprocessor.class);
final HBaseAtlasHook hbaseAtlasHook;
@@ -44,81 +47,38 @@ public class HBaseAtlasCoprocessor extends HBaseAtlasCoprocessorBase {
}
@Override
- public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, HTableDescriptor hTableDescriptor, HRegionInfo[] hRegionInfos) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessoror.postCreateTable()");
- }
- hbaseAtlasHook.sendHBaseTableOperation(hTableDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_TABLE);
+ public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableDescriptor tableDescriptor, RegionInfo[] hRegionInfos) throws IOException {
+ LOG.info("==> HBaseAtlasCoprocessor.postCreateTable()");
+
+ hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_TABLE, observerContext);
if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessoror.postCreateTable()");
+ LOG.debug("<== HBaseAtlasCoprocessor.postCreateTable()");
}
}
@Override
public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteTable()");
- }
- hbaseAtlasHook.sendHBaseTableOperation(null, tableName, HBaseAtlasHook.OPERATION.DELETE_TABLE);
+ LOG.info("==> HBaseAtlasCoprocessor.postDeleteTable()");
+ hbaseAtlasHook.sendHBaseTableOperation(null, tableName, HBaseAtlasHook.OPERATION.DELETE_TABLE, observerContext);
if (LOG.isDebugEnabled()) {
LOG.debug("<== HBaseAtlasCoprocessor.postDeleteTable()");
}
}
@Override
- public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HTableDescriptor hTableDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyTable()");
- }
- hbaseAtlasHook.sendHBaseTableOperation(hTableDescriptor, tableName, HBaseAtlasHook.OPERATION.ALTER_TABLE);
+ public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, TableDescriptor tableDescriptor) throws IOException {
+ LOG.info("==> HBaseAtlasCoprocessor.postModifyTable()");
+ hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, tableName, HBaseAtlasHook.OPERATION.ALTER_TABLE, observerContext);
if (LOG.isDebugEnabled()) {
LOG.debug("<== HBaseAtlasCoprocessor.postModifyTable()");
}
}
@Override
- public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postAddColumn()");
- }
- hbaseAtlasHook.sendHBaseColumnFamilyOperation(hColumnDescriptor, tableName, null, HBaseAtlasHook.OPERATION.CREATE_COLUMN_FAMILY);
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postAddColumn()");
- }
- }
-
- @Override
- public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyColumn()");
- }
- hbaseAtlasHook.sendHBaseColumnFamilyOperation(hColumnDescriptor, tableName, null, HBaseAtlasHook.OPERATION.ALTER_COLUMN_FAMILY);
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postModifyColumn()");
- }
- }
-
- @Override
- public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, byte[] bytes) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteColumn()");
- }
-
- String columnFamily = Bytes.toString(bytes);
- hbaseAtlasHook.sendHBaseColumnFamilyOperation(null, tableName, columnFamily, HBaseAtlasHook.OPERATION.DELETE_COLUMN_FAMILY);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessor.postDeleteColumn()");
- }
- }
-
- @Override
public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postCreateNamespace()");
- }
+ LOG.info("==> HBaseAtlasCoprocessor.postCreateNamespace()");
- hbaseAtlasHook.sendHBaseNameSpaceOperation(namespaceDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_NAMESPACE);
+ hbaseAtlasHook.sendHBaseNameSpaceOperation(namespaceDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_NAMESPACE, observerContext);
if (LOG.isDebugEnabled()) {
LOG.debug("<== HBaseAtlasCoprocessor.postCreateNamespace()");
@@ -127,11 +87,9 @@ public class HBaseAtlasCoprocessor extends HBaseAtlasCoprocessorBase {
@Override
public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postDeleteNamespace()");
- }
+ LOG.info("==> HBaseAtlasCoprocessor.postDeleteNamespace()");
- hbaseAtlasHook.sendHBaseNameSpaceOperation(null, s, HBaseAtlasHook.OPERATION.DELETE_NAMESPACE);
+ hbaseAtlasHook.sendHBaseNameSpaceOperation(null, s, HBaseAtlasHook.OPERATION.DELETE_NAMESPACE, observerContext);
if (LOG.isDebugEnabled()) {
LOG.debug("==> HBaseAtlasCoprocessor.postDeleteNamespace()");
@@ -140,11 +98,9 @@ public class HBaseAtlasCoprocessor extends HBaseAtlasCoprocessorBase {
@Override
public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postModifyNamespace()");
- }
+ LOG.info("==> HBaseAtlasCoprocessor.postModifyNamespace()");
- hbaseAtlasHook.sendHBaseNameSpaceOperation(namespaceDescriptor, null, HBaseAtlasHook.OPERATION.ALTER_NAMESPACE);
+ hbaseAtlasHook.sendHBaseNameSpaceOperation(namespaceDescriptor, null, HBaseAtlasHook.OPERATION.ALTER_NAMESPACE, observerContext);
if (LOG.isDebugEnabled()) {
LOG.debug("<== HBaseAtlasCoprocessor.postModifyNamespace()");
@@ -152,23 +108,22 @@ public class HBaseAtlasCoprocessor extends HBaseAtlasCoprocessorBase {
}
@Override
- public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessoror.postCloneSnapshot()");
- }
- hbaseAtlasHook.sendHBaseTableOperation(hTableDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_TABLE);
+ public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ LOG.info("==> HBaseAtlasCoprocessor.postCloneSnapshot()");
+
+ hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, null, HBaseAtlasHook.OPERATION.CREATE_TABLE, observerContext);
+
if (LOG.isDebugEnabled()) {
- LOG.debug("<== HBaseAtlasCoprocessoror.postCloneSnapshot()");
+ LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()");
}
-
}
@Override
- public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");
- }
- hbaseAtlasHook.sendHBaseTableOperation(hTableDescriptor, hTableDescriptor.getTableName(), HBaseAtlasHook.OPERATION.ALTER_TABLE);
+ public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
+ LOG.info("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");
+
+ hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, snapshot.getTableName(), HBaseAtlasHook.OPERATION.ALTER_TABLE, observerContext);
+
if (LOG.isDebugEnabled()) {
LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()");
}
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessorBase.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessorBase.java
deleted file mode 100644
index f4ca25a..0000000
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/hook/HBaseAtlasCoprocessorBase.java
+++ /dev/null
@@ -1,991 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.hbase.hook;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-
-
-import com.google.common.collect.ImmutableList;
-import org.apache.atlas.hook.AtlasHook;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.*;
-import org.apache.hadoop.hbase.coprocessor.*;
-import org.apache.hadoop.hbase.filter.ByteArrayComparable;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
-import org.apache.hadoop.hbase.io.Reference;
-import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
-import org.apache.hadoop.hbase.protobuf.generated.*;
-import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.regionserver.*;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.wal.WALKey;
-
-
-/**
- * This class exists only to prevent the clutter of methods that we don't intend to implement in the main co-processor class.
- *
- */
-public abstract class HBaseAtlasCoprocessorBase implements MasterObserver,RegionObserver,RegionServerObserver,BulkLoadObserver {
-
- @Override
- public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, HTableDescriptor hTableDescriptor, HRegionInfo[] hRegionInfos) throws IOException {
-
- }
-
- @Override
- public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, HTableDescriptor hTableDescriptor, HRegionInfo[] hRegionInfos) throws IOException {
-
- }
-
- @Override
- public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preTruncateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void preModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, byte[] bytes) throws IOException {
-
- }
-
- @Override
- public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, byte[] bytes) throws IOException {
-
- }
-
- @Override
- public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preMove(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, ServerName serverName, ServerName serverName1) throws IOException {
-
- }
-
-
- @Override
- public void preListProcedures(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preAssign(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo) throws IOException {
-
- }
-
- @Override
- public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, boolean b) throws IOException {
-
- }
-
- @Override
- public void preRegionOffline(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo) throws IOException {
-
- }
-
- @Override
- public void preBalance(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> observerContext, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preMasterInitialization(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription) throws IOException {
-
- }
-
- @Override
- public void preCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void preRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription) throws IOException {
-
- }
-
- @Override
- public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<TableName> list, List<HTableDescriptor> list1) throws IOException {
-
- }
-
- @Override
- public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<TableName> list, List<HTableDescriptor> list1, String s) throws IOException {
-
- }
-
- @Override
- public void preGetTableNames(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<HTableDescriptor> list, String s) throws IOException {
-
- }
-
- @Override
- public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
-
- }
-
- @Override
- public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s) throws IOException {
-
- }
-
- @Override
- public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
-
- }
-
- @Override
- public void preGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s) throws IOException {
-
- }
-
- @Override
- public void preListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<NamespaceDescriptor> list) throws IOException {
-
- }
-
- @Override
- public void preTableFlush(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, TableName tableName, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void preSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, String s1, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void preSetTableQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void preSetNamespaceQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void start(CoprocessorEnvironment coprocessorEnvironment) throws IOException {
-
- }
-
- @Override
- public void stop(CoprocessorEnvironment coprocessorEnvironment) throws IOException {
-
- }
-
- @Override
- public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<HTableDescriptor> list) throws IOException {
-
- }
-
- @Override
- public void postBalance(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<RegionPlan> list) throws IOException {
-
- }
-
- @Override
- public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> observerContext, boolean b, boolean b1) throws IOException {
-
- }
-
- @Override
- public void postGetNamespaceDescriptor(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
-
- }
-
- @Override
- public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void postSetNamespaceQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void postAbortProcedure(ObserverContext<MasterCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postListProcedures(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<ProcedureInfo> list) throws IOException {
-
- }
-
- @Override
- public void postCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, HTableDescriptor hTableDescriptor, HRegionInfo[] hRegionInfos) throws IOException {
-
- }
-
- @Override
- public void postDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postTruncateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postTruncateTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void postModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, byte[] bytes) throws IOException {
-
- }
- @Override
- public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postMove(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, ServerName serverName, ServerName serverName1) throws IOException {
-
- }
-
- @Override
- public void postAssign(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo) throws IOException {
-
- }
-
- @Override
- public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, boolean b) throws IOException {
-
- }
-
- @Override
- public void postRegionOffline(ObserverContext<MasterCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo) throws IOException {
-
- }
-
- @Override
- public void postListSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription) throws IOException {
-
- }
-
- @Override
- public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, HBaseProtos.SnapshotDescription snapshotDescription) throws IOException {
-
- }
-
- @Override
- public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<TableName> list, List<HTableDescriptor> list1, String s) throws IOException {
-
- }
-
- @Override
- public void postGetTableNames(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<HTableDescriptor> list, String s) throws IOException {
-
- }
-
- @Override
- public void postListNamespaceDescriptors(ObserverContext<MasterCoprocessorEnvironment> observerContext, List<NamespaceDescriptor> list) throws IOException {
-
- }
-
- @Override
- public void postTableFlush(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, QuotaProtos.Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, TableName tableName, QuotaProtos.Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void postSetUserQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s, String s1, QuotaProtos.Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void postSetTableQuota(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, QuotaProtos.Quotas quotas) throws IOException {
-
- }
-
- @Override
- public void preOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext) {
-
- }
-
- @Override
- public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> observerContext) {
-
- }
-
- @Override
- public InternalScanner preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, KeyValueScanner keyValueScanner, InternalScanner internalScanner) throws IOException {
- return null;
- }
-
- @Override
- public void preFlush(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public InternalScanner preFlush(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, InternalScanner internalScanner) throws IOException {
- return internalScanner;
- }
-
- @Override
- public void postFlush(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postFlush(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, StoreFile storeFile) throws IOException {
-
- }
-
- @Override
- public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, List<StoreFile> list, CompactionRequest compactionRequest) throws IOException {
-
- }
-
- @Override
- public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, List<StoreFile> list) throws IOException {
-
- }
-
- @Override
- public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, ImmutableList<StoreFile> immutableList, CompactionRequest compactionRequest) {
-
- }
-
- @Override
- public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, ImmutableList<StoreFile> immutableList) {
-
- }
-
- @Override
- public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, InternalScanner internalScanner, ScanType scanType, CompactionRequest compactionRequest) throws IOException {
- return internalScanner;
- }
-
- @Override
- public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, InternalScanner internalScanner, ScanType scanType) throws IOException {
- return internalScanner;
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, List<? extends KeyValueScanner> list, ScanType scanType, long l, InternalScanner internalScanner, CompactionRequest compactionRequest) throws IOException {
- return internalScanner;
- }
-
- @Override
- public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, List<? extends KeyValueScanner> list, ScanType scanType, long l, InternalScanner internalScanner) throws IOException {
- return internalScanner;
- }
-
- @Override
- public void postCompact(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, StoreFile storeFile, CompactionRequest compactionRequest) throws IOException {
-
- }
-
- @Override
- public void postCompact(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, StoreFile storeFile) throws IOException {
-
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes) throws IOException {
-
- }
-
- @Override
- public void postSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext, Region region, Region region1) throws IOException {
-
- }
-
- @Override
- public void preSplitBeforePONR(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, List<Mutation> list) throws IOException {
-
- }
-
- @Override
- public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preRollBackSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postRollBackSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postCompleteSplit(ObserverContext<RegionCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preClose(ObserverContext<RegionCoprocessorEnvironment> observerContext, boolean b) throws IOException {
-
- }
-
- @Override
- public void postClose(ObserverContext<RegionCoprocessorEnvironment> observerContext, boolean b) {
-
- }
-
- @Override
- public void preGetClosestRowBefore(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, Result result) throws IOException {
-
- }
-
- @Override
- public void postGetClosestRowBefore(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, Result result) throws IOException {
-
- }
-
- @Override
- public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> observerContext, Get get, List<Cell> list) throws IOException {
-
- }
-
- @Override
- public void postGetOp(ObserverContext<RegionCoprocessorEnvironment> observerContext, Get get, List<Cell> list) throws IOException {
-
- }
-
- @Override
- public boolean preExists(ObserverContext<RegionCoprocessorEnvironment> observerContext, Get get, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean postExists(ObserverContext<RegionCoprocessorEnvironment> observerContext, Get get, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public void prePut(ObserverContext<RegionCoprocessorEnvironment> observerContext, Put put, WALEdit walEdit, Durability durability) throws IOException {
-
- }
-
- @Override
- public void postPut(ObserverContext<RegionCoprocessorEnvironment> observerContext, Put put, WALEdit walEdit, Durability durability) throws IOException {
-
- }
-
- @Override
- public void preDelete(ObserverContext<RegionCoprocessorEnvironment> observerContext, Delete delete, WALEdit walEdit, Durability durability) throws IOException {
-
- }
-
- @Override
- public void prePrepareTimeStampForDeleteVersion(ObserverContext<RegionCoprocessorEnvironment> observerContext, Mutation mutation, Cell cell, byte[] bytes, Get get) throws IOException {
-
- }
-
- @Override
- public void postDelete(ObserverContext<RegionCoprocessorEnvironment> observerContext, Delete delete, WALEdit walEdit, Durability durability) throws IOException {
-
- }
-
- @Override
- public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> observerContext, MiniBatchOperationInProgress<Mutation> miniBatchOperationInProgress) throws IOException {
-
- }
-
- @Override
- public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> observerContext, MiniBatchOperationInProgress<Mutation> miniBatchOperationInProgress) throws IOException {
-
- }
-
- @Override
- public void postStartRegionOperation(ObserverContext<RegionCoprocessorEnvironment> observerContext, Region.Operation operation) throws IOException {
-
- }
-
- @Override
- public void postCloseRegionOperation(ObserverContext<RegionCoprocessorEnvironment> observerContext, Region.Operation operation) throws IOException {
-
- }
-
- @Override
- public void postBatchMutateIndispensably(ObserverContext<RegionCoprocessorEnvironment> observerContext, MiniBatchOperationInProgress<Mutation> miniBatchOperationInProgress, boolean b) throws IOException {
-
- }
-
- @Override
- public boolean preCheckAndPut(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Put put, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean preCheckAndPutAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Put put, boolean b) throws IOException {
- return false;
- }
-
- @Override
- public boolean postCheckAndPut(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Put put, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean preCheckAndDelete(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Delete delete, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean preCheckAndDeleteAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Delete delete, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean postCheckAndDelete(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, CompareFilter.CompareOp compareOp, ByteArrayComparable byteArrayComparable, Delete delete, boolean b) throws IOException {
- return false;
- }
-
- @Override
- public long preIncrementColumnValue(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, long l, boolean b) throws IOException {
- return l;
- }
-
- @Override
- public long postIncrementColumnValue(ObserverContext<RegionCoprocessorEnvironment> observerContext, byte[] bytes, byte[] bytes1, byte[] bytes2, long l, boolean b, long l1) throws IOException {
- return l;
- }
-
- @Override
- public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> observerContext, Append append) throws IOException {
- return null;
- }
-
- @Override
- public Result preAppendAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> observerContext, Append append) throws IOException {
- return null;
- }
-
- @Override
- public Result postAppend(ObserverContext<RegionCoprocessorEnvironment> observerContext, Append append, Result result) throws IOException {
- return result;
- }
-
- @Override
- public Result preIncrement(ObserverContext<RegionCoprocessorEnvironment> observerContext, Increment increment) throws IOException {
- return null;
- }
-
- @Override
- public Result preIncrementAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> observerContext, Increment increment) throws IOException {
- return null;
- }
-
- @Override
- public Result postIncrement(ObserverContext<RegionCoprocessorEnvironment> observerContext, Increment increment, Result result) throws IOException {
- return result;
- }
-
- @Override
- public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Scan scan, RegionScanner regionScanner) throws IOException {
- return regionScanner;
- }
-
- @Override
- public KeyValueScanner preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Store store, Scan scan, NavigableSet<byte[]> navigableSet, KeyValueScanner keyValueScanner) throws IOException {
- return keyValueScanner;
- }
-
- @Override
- public RegionScanner postScannerOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, Scan scan, RegionScanner regionScanner) throws IOException {
- return regionScanner;
- }
-
- @Override
- public boolean preScannerNext(ObserverContext<RegionCoprocessorEnvironment> observerContext, InternalScanner internalScanner, List<Result> list, int i, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean postScannerNext(ObserverContext<RegionCoprocessorEnvironment> observerContext, InternalScanner internalScanner, List<Result> list, int i, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public boolean postScannerFilterRow(ObserverContext<RegionCoprocessorEnvironment> observerContext, InternalScanner internalScanner, byte[] bytes, int i, short i1, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public void preScannerClose(ObserverContext<RegionCoprocessorEnvironment> observerContext, InternalScanner internalScanner) throws IOException {
-
- }
-
- @Override
- public void postScannerClose(ObserverContext<RegionCoprocessorEnvironment> observerContext, InternalScanner internalScanner) throws IOException {
-
- }
-
- @Override
- public void preWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, WALKey walKey, WALEdit walEdit) throws IOException {
-
- }
-
- @Override
- public void preWALRestore(ObserverContext<RegionCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, HLogKey hLogKey, WALEdit walEdit) throws IOException {
-
- }
-
- @Override
- public void postWALRestore(ObserverContext<? extends RegionCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, WALKey walKey, WALEdit walEdit) throws IOException {
-
- }
-
- @Override
- public void postWALRestore(ObserverContext<RegionCoprocessorEnvironment> observerContext, HRegionInfo hRegionInfo, HLogKey hLogKey, WALEdit walEdit) throws IOException {
-
- }
-
- @Override
- public void preBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> observerContext, List<Pair<byte[], String>> list) throws IOException {
-
- }
-
- @Override
- public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> observerContext, List<Pair<byte[], String>> list, boolean b) throws IOException {
- return b;
- }
-
- @Override
- public StoreFile.Reader preStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, FileSystem fileSystem, Path path, FSDataInputStreamWrapper fsDataInputStreamWrapper, long l, CacheConfig cacheConfig, Reference reference, StoreFile.Reader reader) throws IOException {
- return reader;
- }
-
- @Override
- public StoreFile.Reader postStoreFileReaderOpen(ObserverContext<RegionCoprocessorEnvironment> observerContext, FileSystem fileSystem, Path path, FSDataInputStreamWrapper fsDataInputStreamWrapper, long l, CacheConfig cacheConfig, Reference reference, StoreFile.Reader reader) throws IOException {
- return reader;
- }
-
- @Override
- public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> observerContext, MutationType mutationType, Mutation mutation, Cell cell, Cell cell1) throws IOException {
- return cell;
- }
-
- @Override
- public DeleteTracker postInstantiateDeleteTracker(ObserverContext<RegionCoprocessorEnvironment> observerContext, DeleteTracker deleteTracker) throws IOException {
- return deleteTracker;
- }
-
- @Override
- public void preStopRegionServer(ObserverContext<RegionServerCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void preMerge(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1) throws IOException {
-
- }
-
- @Override
- public void postMerge(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1, Region region2) throws IOException {
-
- }
-
- @Override
- public void preMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1, @MetaMutationAnnotation List<Mutation> list) throws IOException {
-
- }
-
- @Override
- public void postMergeCommit(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1, Region region2) throws IOException {
-
- }
-
- @Override
- public void preRollBackMerge(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1) throws IOException {
-
- }
-
- @Override
- public void postRollBackMerge(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, Region region, Region region1) throws IOException {
-
- }
-
- @Override
- public void preRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public void postRollWALWriterRequest(ObserverContext<RegionServerCoprocessorEnvironment> observerContext) throws IOException {
-
- }
-
- @Override
- public ReplicationEndpoint postCreateReplicationEndPoint(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, ReplicationEndpoint replicationEndpoint) {
- return null;
- }
-
- @Override
- public void preReplicateLogEntries(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, List<AdminProtos.WALEntry> list, CellScanner cellScanner) throws IOException {
-
- }
-
- @Override
- public void postReplicateLogEntries(ObserverContext<RegionServerCoprocessorEnvironment> observerContext, List<AdminProtos.WALEntry> list, CellScanner cellScanner) throws IOException {
-
- }
-
- @Override
- public void prePrepareBulkLoad(ObserverContext<RegionCoprocessorEnvironment> observerContext, SecureBulkLoadProtos.PrepareBulkLoadRequest prepareBulkLoadRequest) throws IOException {
-
- }
-
- @Override
- public void preCleanupBulkLoad(ObserverContext<RegionCoprocessorEnvironment> observerContext, SecureBulkLoadProtos.CleanupBulkLoadRequest cleanupBulkLoadRequest) throws IOException {
-
- }
-
- @Override
- public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, HTableDescriptor hTableDescriptor, HRegionInfo[] hRegionInfos) throws IOException {
-
- }
-
-
-
- @Override
- public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName) throws IOException {
-
- }
-
- @Override
- public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HTableDescriptor hTableDescriptor) throws IOException {
-
- }
-
- @Override
- public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, HColumnDescriptor hColumnDescriptor) throws IOException {
-
- }
-
- @Override
- public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> observerContext, TableName tableName, byte[] bytes) throws IOException {
-
- }
-
- @Override
- public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
-
- }
-
- @Override
- public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, String s) throws IOException {
-
- }
-
- @Override
- public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> observerContext, NamespaceDescriptor namespaceDescriptor) throws IOException {
-
- }
-
- @Override
- public void preAbortProcedure(ObserverContext<MasterCoprocessorEnvironment> observerContext, ProcedureExecutor<MasterProcedureEnv> procedureExecutor, long l) throws IOException {
-
- }
-}
diff --git a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/model/HBaseOperationContext.java b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/model/HBaseOperationContext.java
index bc8485b..1ef7c07 100644
--- a/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/model/HBaseOperationContext.java
+++ b/addons/hbase-bridge/src/main/java/org/apache/atlas/hbase/model/HBaseOperationContext.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.security.UserGroupInformation;
import java.util.ArrayList;
@@ -37,41 +39,41 @@ public class HBaseOperationContext {
private final HBaseAtlasHook.OPERATION operation;
private final String user;
private final NamespaceDescriptor namespaceDescriptor;
- private final HTableDescriptor hTableDescriptor;
- private final HColumnDescriptor[] hColumnDescriptors;
+ private final TableDescriptor tableDescriptor;
+ private final ColumnFamilyDescriptor[] columnFamilyDescriptors;
private final TableName tableName;
private final String nameSpace;
private final String columnFamily;
private final String owner;
- private final HColumnDescriptor hColumnDescriptor;
-
- public HBaseOperationContext(NamespaceDescriptor namespaceDescriptor, String nameSpace, HTableDescriptor hTableDescriptor, TableName tableName, HColumnDescriptor[] hColumnDescriptors,
- HColumnDescriptor hColumnDescriptor, String columnFamily, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi , String user, String owner,
- Map<String, String> hbaseConf) {
- this.namespaceDescriptor = namespaceDescriptor;
- this.nameSpace = nameSpace;
- this.hTableDescriptor = hTableDescriptor;
- this.tableName = tableName;
- this.hColumnDescriptors = hColumnDescriptors;
- this.hColumnDescriptor = hColumnDescriptor;
- this.columnFamily = columnFamily;
- this.operation = operation;
- this.ugi = ugi;
- this.user = user;
- this.owner = owner;
- this.hbaseConf = hbaseConf;
+ private final ColumnFamilyDescriptor columnFamilyDescriptor;
+
+ public HBaseOperationContext(NamespaceDescriptor namespaceDescriptor, String nameSpace, TableDescriptor tableDescriptor, TableName tableName, ColumnFamilyDescriptor[] columnFamilyDescriptors,
+ ColumnFamilyDescriptor columnFamilyDescriptor, String columnFamily, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi , String user, String owner,
+ Map<String, String> hbaseConf) {
+ this.namespaceDescriptor = namespaceDescriptor;
+ this.nameSpace = nameSpace;
+ this.tableDescriptor = tableDescriptor;
+ this.tableName = tableName;
+ this.columnFamilyDescriptors = columnFamilyDescriptors;
+ this.columnFamilyDescriptor = columnFamilyDescriptor;
+ this.columnFamily = columnFamily;
+ this.operation = operation;
+ this.ugi = ugi;
+ this.user = user;
+ this.owner = owner;
+ this.hbaseConf = hbaseConf;
}
public HBaseOperationContext(NamespaceDescriptor namespaceDescriptor, String nameSpace, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi , String user, String owner) {
this(namespaceDescriptor, nameSpace, null, null, null, null, null, operation, ugi, user, owner, null);
}
- public HBaseOperationContext(String nameSpace, HTableDescriptor hTableDescriptor, TableName tableName, HColumnDescriptor[] hColumnDescriptor, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi, String user, String owner, Map<String,String> hbaseConf) {
- this(null, nameSpace, hTableDescriptor, tableName, hColumnDescriptor, null, null, operation, ugi, user, owner, hbaseConf);
+ public HBaseOperationContext(String nameSpace, TableDescriptor tableDescriptor, TableName tableName, ColumnFamilyDescriptor[] columnFamilyDescriptors, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi, String user, String owner, Map<String,String> hbaseConf) {
+ this(null, nameSpace, tableDescriptor, tableName, columnFamilyDescriptors, null, null, operation, ugi, user, owner, hbaseConf);
}
- public HBaseOperationContext(String nameSpace, TableName tableName, HColumnDescriptor hColumnDescriptor, String columnFamily, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi, String user, String owner, Map<String,String> hbaseConf) {
- this(null, nameSpace, null, tableName, null, hColumnDescriptor, columnFamily, operation, ugi, user, owner, hbaseConf);
+ public HBaseOperationContext(String nameSpace, TableName tableName, ColumnFamilyDescriptor columnFamilyDescriptor, String columnFamily, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi, String user, String owner, Map<String,String> hbaseConf) {
+ this(null, nameSpace, null, tableName, null, columnFamilyDescriptor, columnFamily, operation, ugi, user, owner, hbaseConf);
}
private List<HookNotification> messages = new ArrayList<>();
@@ -96,12 +98,12 @@ public class HBaseOperationContext {
return namespaceDescriptor;
}
- public HTableDescriptor gethTableDescriptor() {
- return hTableDescriptor;
+ public TableDescriptor gethTableDescriptor() {
+ return tableDescriptor;
}
- public HColumnDescriptor[] gethColumnDescriptors() {
- return hColumnDescriptors;
+ public ColumnFamilyDescriptor[] gethColumnDescriptors() {
+ return columnFamilyDescriptors;
}
public TableName getTableName() {
@@ -112,8 +114,8 @@ public class HBaseOperationContext {
return nameSpace;
}
- public HColumnDescriptor gethColumnDescriptor() {
- return hColumnDescriptor;
+ public ColumnFamilyDescriptor gethColumnDescriptor() {
+ return columnFamilyDescriptor;
}
public String getColummFamily() {
@@ -153,15 +155,15 @@ public class HBaseOperationContext {
if (tableName != null ) {
sb.append("Table={").append(tableName).append("}");
} else {
- if ( hColumnDescriptor != null) {
- sb.append("Table={").append(hTableDescriptor.toString()).append("}");
+ if ( columnFamilyDescriptor != null) {
+ sb.append("Table={").append(tableDescriptor.toString()).append("}");
}
}
if (columnFamily != null ) {
sb.append("Columm Family={").append(columnFamily).append("}");
} else {
- if ( hColumnDescriptor != null) {
- sb.append("Columm Family={").append(hColumnDescriptor.toString()).append("}");
+ if ( columnFamilyDescriptor != null) {
+ sb.append("Columm Family={").append(columnFamilyDescriptor.toString()).append("}");
}
}
sb.append("Message ={").append(getMessages()).append("} ");
diff --git a/addons/hbase-bridge/src/test/java/org/apache/atlas/hbase/HBaseAtlasHookIT.java b/addons/hbase-bridge/src/test/java/org/apache/atlas/hbase/HBaseAtlasHookIT.java
index 15bfbe3..e346788 100644
--- a/addons/hbase-bridge/src/test/java/org/apache/atlas/hbase/HBaseAtlasHookIT.java
+++ b/addons/hbase-bridge/src/test/java/org/apache/atlas/hbase/HBaseAtlasHookIT.java
@@ -44,9 +44,11 @@ import java.io.IOException;
import java.net.ServerSocket;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.fail;
+import static org.testng.AssertJUnit.assertFalse;
public class HBaseAtlasHookIT {
@@ -76,6 +78,12 @@ public class HBaseAtlasHookIT {
}
@Test
+ public void testGetMetaTableRows() throws Exception {
+ List<byte[]> results = utility.getMetaTableRows();
+ assertFalse("results should have some entries and is empty.", results.isEmpty());
+ }
+
+ @Test (enabled = false)
public void testCreateNamesapce() throws Exception {
final Configuration conf = HBaseConfiguration.create();
@@ -103,7 +111,7 @@ public class HBaseAtlasHookIT {
}
}
- @Test
+ @Test (enabled = false)
public void testCreateTable() throws Exception {
final Configuration conf = HBaseConfiguration.create();
@@ -194,8 +202,7 @@ public class HBaseAtlasHookIT {
utility.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(getFreePort()));
utility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure");
utility.getConfiguration().set("hbase.table.sanity.checks", "false");
- utility.getConfiguration().set("hbase.coprocessor.master.classes",
- "org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor");
+ utility.getConfiguration().set("hbase.coprocessor.master.classes", "org.apache.atlas.hbase.hook.HBaseAtlasCoprocessor");
utility.startMiniCluster();
}
@@ -252,7 +259,7 @@ public class HBaseAtlasHookIT {
protected String assertEntityIsRegistered(final String typeName, final String property, final String value,
final HBaseAtlasHookIT.AssertPredicate assertPredicate) throws Exception {
- waitFor(80000, new HBaseAtlasHookIT.Predicate() {
+ waitFor(30000, new HBaseAtlasHookIT.Predicate() {
@Override
public void evaluate() throws Exception {
AtlasEntityWithExtInfo entity = atlasClient.getEntityByAttribute(typeName, Collections.singletonMap(property, value));
diff --git a/addons/hbase-bridge/src/test/resources/atlas-log4j.xml b/addons/hbase-bridge/src/test/resources/atlas-log4j.xml
index c183871..2c9815f 100755
--- a/addons/hbase-bridge/src/test/resources/atlas-log4j.xml
+++ b/addons/hbase-bridge/src/test/resources/atlas-log4j.xml
@@ -32,8 +32,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
@@ -42,8 +40,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
@@ -52,7 +48,14 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
- <param name="maxFileSize" value="100MB" />
+ </layout>
+ </appender>
+
+ <appender name="HBASE" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/hbase.log"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %x %m%n"/>
</layout>
</appender>
@@ -61,8 +64,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %m"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
@@ -88,6 +89,11 @@
<appender-ref ref="FILE"/>
</logger>
+ <logger name="org.apache.hadoop" additivity="false">
+ <level value="debug"/>
+ <appender-ref ref="HBASE"/>
+ </logger>
+
<logger name="org.janusgraph" additivity="false">
<level value="warn"/>
<appender-ref ref="FILE"/>
diff --git a/addons/hbase-testing-util/pom.xml b/addons/hbase-testing-util/pom.xml
new file mode 100644
index 0000000..ef6d4ad
--- /dev/null
+++ b/addons/hbase-testing-util/pom.xml
@@ -0,0 +1,203 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ <relativePath>../../</relativePath>
+ </parent>
+ <artifactId>hbase-testing-util</artifactId>
+ <name>Apache HBase - Testing Util</name>
+ <description>HBase Testing Utilities.</description>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hadoop.version>3.0.3</hadoop.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-zookeeper</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <version>${hbase.version}</version>
+ <type>test-jar</type>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-protocol</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+
+
+
+
+ </dependencies>
+
+</project>
diff --git a/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java b/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java
new file mode 100644
index 0000000..0beb035
--- /dev/null
+++ b/addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.hbase;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+import static org.testng.AssertJUnit.assertFalse;
+
+
+/**
+ * Make sure we can spin up a HBTU without a hbase-site.xml
+ */
+public class TestHBaseTestingUtilSpinup {
+ private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTestingUtilSpinup.class);
+ private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ UTIL.startMiniCluster();
+ if (!UTIL.getHBaseCluster().waitForActiveAndReadyMaster(30000)) {
+ throw new RuntimeException("Active master not ready");
+ }
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testGetMetaTableRows() throws Exception {
+ List<byte[]> results = UTIL.getMetaTableRows();
+ assertFalse("results should have some entries and is empty.", results.isEmpty());
+ }
+
+}
diff --git a/addons/hbase-bridge/src/test/resources/atlas-log4j.xml b/addons/hbase-testing-util/src/test/resources/atlas-log4j.xml
similarity index 92%
copy from addons/hbase-bridge/src/test/resources/atlas-log4j.xml
copy to addons/hbase-testing-util/src/test/resources/atlas-log4j.xml
index c183871..47d4d5c 100755
--- a/addons/hbase-bridge/src/test/resources/atlas-log4j.xml
+++ b/addons/hbase-testing-util/src/test/resources/atlas-log4j.xml
@@ -32,8 +32,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
@@ -42,8 +40,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
@@ -52,7 +48,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %x %m%n"/>
- <param name="maxFileSize" value="100MB" />
</layout>
</appender>
@@ -61,8 +56,6 @@
<param name="Append" value="true"/>
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %m"/>
- <param name="maxFileSize" value="100MB" />
- <param name="maxBackupIndex" value="20" />
</layout>
</appender>
diff --git a/addons/hive-bridge-shim/pom.xml b/addons/hive-bridge-shim/pom.xml
index e7e40cc..0ce70cd 100755
--- a/addons/hive-bridge-shim/pom.xml
+++ b/addons/hive-bridge-shim/pom.xml
@@ -30,11 +30,6 @@
<name>Apache Atlas Hive Bridge Shim</name>
<packaging>jar</packaging>
- <properties>
- <hive.version>1.2.1</hive.version>
- <calcite.version>0.9.2-incubating</calcite.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 5ffecac..b2ff979 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -30,11 +30,6 @@
<name>Apache Atlas Hive Bridge</name>
<packaging>jar</packaging>
- <properties>
- <hive.version>1.2.1</hive.version>
- <calcite.version>0.9.2-incubating</calcite.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
@@ -57,6 +52,10 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -66,6 +65,12 @@
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
@@ -76,7 +81,15 @@
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -136,6 +149,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -265,11 +282,6 @@
<version>${jersey.version}</version>
</artifactItem>
<artifactItem>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </artifactItem>
- <artifactItem>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
@@ -387,7 +399,7 @@
</systemProperty>
<systemProperty>
<name>log4j.configuration</name>
- <value>file:///${project.build.directory}/test-classes/atlas-log4j.xml</value>
+ <value>file:///${project.build.directory}/../../../distro/src/conf/atlas-log4j.xml</value>
</systemProperty>
<systemProperty>
<name>atlas.graphdb.backend</name>
@@ -401,7 +413,22 @@
<stopKey>atlas-stop</stopKey>
<stopPort>31001</stopPort>
<stopWait>${jetty-maven-plugin.stopWait}</stopWait>
+ <daemon>${debug.jetty.daemon}</daemon>
+ <testClassesDirectory>${project.build.testOutputDirectory}</testClassesDirectory>
+ <useTestClasspath>true</useTestClasspath>
</configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>2.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>2.8</version>
+ </dependency>
+ </dependencies>
<executions>
<execution>
<id>start-jetty</id>
@@ -409,9 +436,6 @@
<goals>
<goal>deploy-war</goal>
</goals>
- <configuration>
- <daemon>true</daemon>
- </configuration>
</execution>
<execution>
<id>stop-jetty</id>
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index 6acf89d..ebc5d70 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -149,7 +149,6 @@ public class HiveITBase {
protected void runCommandWithDelay(Driver driver, String cmd, int sleepMs) throws Exception {
LOG.debug("Running command '{}'", cmd);
- ss.setCommandType(null);
CommandProcessorResponse response = driver.run(cmd);
assertEquals(response.getResponseCode(), 0);
if (sleepMs != 0) {
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index a36a666..142e355 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -57,6 +57,7 @@ import org.testng.Assert;
import org.testng.annotations.Test;
import java.io.File;
+import java.nio.file.Files;
import java.text.ParseException;
import java.util.*;
@@ -196,14 +197,12 @@ public class HiveHookIT extends HiveITBase {
}
private Set<ReadEntity> getInputs(String inputName, Entity.Type entityType) throws HiveException {
- final ReadEntity entity = new ReadEntity();
+ final ReadEntity entity;
if (Entity.Type.DFS_DIR.equals(entityType)) {
- entity.setName(lower(new Path(inputName).toString()));
- entity.setTyp(Entity.Type.DFS_DIR);
+ entity = new TestReadEntity(lower(new Path(inputName).toString()), entityType);
} else {
- entity.setName(getQualifiedTblName(inputName));
- entity.setTyp(entityType);
+ entity = new TestReadEntity(getQualifiedTblName(inputName), entityType);
}
if (entityType == Entity.Type.TABLE) {
@@ -214,14 +213,12 @@ public class HiveHookIT extends HiveITBase {
}
private Set<WriteEntity> getOutputs(String inputName, Entity.Type entityType) throws HiveException {
- final WriteEntity entity = new WriteEntity();
+ final WriteEntity entity;
if (Entity.Type.DFS_DIR.equals(entityType) || Entity.Type.LOCAL_DIR.equals(entityType)) {
- entity.setName(lower(new Path(inputName).toString()));
- entity.setTyp(entityType);
+ entity = new TestWriteEntity(lower(new Path(inputName).toString()), entityType);
} else {
- entity.setName(getQualifiedTblName(inputName));
- entity.setTyp(entityType);
+ entity = new TestWriteEntity(getQualifiedTblName(inputName), entityType);
}
if (entityType == Entity.Type.TABLE) {
@@ -591,8 +588,8 @@ public class HiveHookIT extends HiveITBase {
@Test
public void testInsertIntoLocalDir() throws Exception {
String tableName = createTable();
- File randomLocalPath = File.createTempFile("hiverandom", ".tmp");
- String query = "insert overwrite LOCAL DIRECTORY '" + randomLocalPath.getAbsolutePath() + "' select id, name from " + tableName;
+ String randomLocalPath = mkdir("hiverandom.tmp");
+ String query = "insert overwrite LOCAL DIRECTORY '" + randomLocalPath + "' select id, name from " + tableName;
runCommand(query);
@@ -715,7 +712,6 @@ public class HiveHookIT extends HiveITBase {
Set<ReadEntity> inputs = getInputs(tableName, Entity.Type.TABLE);
Set<WriteEntity> outputs = getOutputs(insertTableName, Entity.Type.TABLE);
- outputs.iterator().next().setName(getQualifiedTblName(insertTableName + HiveMetaStoreBridge.TEMP_TABLE_PREFIX + SessionState.get().getSessionId()));
outputs.iterator().next().setWriteType(WriteEntity.WriteType.INSERT);
validateProcess(constructEvent(query, HiveOperation.QUERY, inputs, outputs));
@@ -1536,19 +1532,13 @@ public class HiveHookIT extends HiveITBase {
}
private WriteEntity getPartitionOutput() {
- WriteEntity partEntity = new WriteEntity();
-
- partEntity.setName(PART_FILE);
- partEntity.setTyp(Entity.Type.PARTITION);
+ TestWriteEntity partEntity = new TestWriteEntity(PART_FILE, Entity.Type.PARTITION);
return partEntity;
}
private ReadEntity getPartitionInput() {
- ReadEntity partEntity = new ReadEntity();
-
- partEntity.setName(PART_FILE);
- partEntity.setTyp(Entity.Type.PARTITION);
+ ReadEntity partEntity = new TestReadEntity(PART_FILE, Entity.Type.PARTITION);
return partEntity;
}
@@ -2056,4 +2046,38 @@ public class HiveHookIT extends HiveITBase {
return tableName;
}
+
+ // ReadEntity class doesn't offer a constructor that takes (name, type). A hack to get the tests going!
+ private static class TestReadEntity extends ReadEntity {
+ private final String name;
+ private final Entity.Type type;
+
+ public TestReadEntity(String name, Entity.Type type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ @Override
+ public String getName() { return name; }
+
+ @Override
+ public Entity.Type getType() { return type; }
+ }
+
+ // WriteEntity class doesn't offer a constructor that takes (name, type). A hack to get the tests going!
+ private static class TestWriteEntity extends WriteEntity {
+ private final String name;
+ private final Entity.Type type;
+
+ public TestWriteEntity(String name, Entity.Type type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ @Override
+ public String getName() { return name; }
+
+ @Override
+ public Entity.Type getType() { return type; }
+ }
}
diff --git a/addons/hive-bridge/src/test/resources/hive-site.xml b/addons/hive-bridge/src/test/resources/hive-site.xml
index ff98668..4605ae3 100644
--- a/addons/hive-bridge/src/test/resources/hive-site.xml
+++ b/addons/hive-bridge/src/test/resources/hive-site.xml
@@ -48,7 +48,7 @@
<property>
<name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:${project.basedir}/target/metastore_db;create=true</value>
+ <value>jdbc:derby:;databaseName=${project.basedir}/target/metastore_db;create=true</value>
</property>
<property>
@@ -70,4 +70,25 @@
<name>hive.zookeeper.quorum</name>
<value>localhost:19026</value>
</property>
+
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.disallow.incompatible.col.type.changes</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.exec.scratchdir</name>
+ <value>${project.basedir}/target/scratchdir</value>
+ </property>
+
</configuration>
\ No newline at end of file
diff --git a/addons/kafka-bridge/pom.xml b/addons/kafka-bridge/pom.xml
index d1e2497..fd5f399 100644
--- a/addons/kafka-bridge/pom.xml
+++ b/addons/kafka-bridge/pom.xml
@@ -44,7 +44,7 @@
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-bundle</artifactId>
- <version>1.19</version>
+ <version>${jersey.version}</version>
<scope>test</scope>
</dependency>
diff --git a/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
index 8624774..14b1fbe 100644
--- a/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
+++ b/addons/models/1000-Hadoop/patches/005-hbase_table_column_family_add_additional_attribute.json
@@ -32,6 +32,14 @@
"isUnique": false
},
{
+ "name": "isNormalizationEnabled",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
"name": "replicasPerRegion",
"typeName": "int",
"cardinality": "SINGLE",
@@ -90,6 +98,14 @@
"isUnique": false
},
{
+ "name": "inMemoryCompactionPolicy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
"name": "keepDeletedCells",
"typeName": "boolean",
"cardinality": "SINGLE",
@@ -122,6 +138,14 @@
"isUnique": false
},
{
+ "name": "StoragePolicy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
"name": "ttl",
"typeName": "int",
"cardinality": "SINGLE",
@@ -176,6 +200,30 @@
"isIndexable": false,
"isOptional": true,
"isUnique": false
+ },
+ {
+ "name": "newVersionBehavior",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isMobEnabled",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "mobCompactPartitionPolicy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
}
]
}
diff --git a/addons/sqoop-bridge-shim/pom.xml b/addons/sqoop-bridge-shim/pom.xml
index f5a0dcf..4439e5c 100755
--- a/addons/sqoop-bridge-shim/pom.xml
+++ b/addons/sqoop-bridge-shim/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Atlas Sqoop Bridge Shim</name>
<packaging>jar</packaging>
- <properties>
- <sqoop.version>1.4.6.2.3.99.0-195</sqoop.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index 1ca5326..edf8377 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -30,12 +30,6 @@
<name>Apache Atlas Sqoop Bridge</name>
<packaging>jar</packaging>
- <properties>
- <!-- maps to 1.4.7-SNAPSHOT version of apache sqoop -->
- <sqoop.version>1.4.6.2.3.99.0-195</sqoop.version>
- <hive.version>1.2.1</hive.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
@@ -80,6 +74,10 @@
<artifactId>*</artifactId>
</exclusion>
<exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
diff --git a/addons/storm-bridge-shim/pom.xml b/addons/storm-bridge-shim/pom.xml
index 5713c8f..b12cc2b 100755
--- a/addons/storm-bridge-shim/pom.xml
+++ b/addons/storm-bridge-shim/pom.xml
@@ -30,10 +30,6 @@
<name>Apache Atlas Storm Bridge Shim</name>
<packaging>jar</packaging>
- <properties>
- <storm.version>1.2.0</storm.version>
- </properties>
-
<dependencies>
<!-- Logging -->
<dependency>
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index d85be86..6425a3a 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -29,11 +29,6 @@
<name>Apache Atlas Storm Bridge</name>
<packaging>jar</packaging>
- <properties>
- <storm.version>1.2.0</storm.version>
- <hive.version>1.2.1</hive.version>
- </properties>
-
<dependencies>
<!-- apache atlas core dependencies -->
<dependency>
@@ -77,6 +72,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
diff --git a/authorization/src/test/resources/atlas-application.properties b/authorization/src/test/resources/atlas-application.properties
index 4d05ae7..2e02678 100644
--- a/authorization/src/test/resources/atlas-application.properties
+++ b/authorization/src/test/resources/atlas-application.properties
@@ -57,7 +57,7 @@ atlas.graph.storage.directory=${sys:atlas.data}/berkley
#hbase
#For standalone mode , specify localhost
-#for distributed mode, specify zookeeper quorum here
+#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=${graph.storage.hostname}
atlas.graph.storage.hbase.regions-per-server=1
diff --git a/build-tools/src/main/resources/checkstyle-suppressions.xml b/build-tools/src/main/resources/checkstyle-suppressions.xml
index 0025360..759a52e 100644
--- a/build-tools/src/main/resources/checkstyle-suppressions.xml
+++ b/build-tools/src/main/resources/checkstyle-suppressions.xml
@@ -23,4 +23,7 @@
<suppressions>
<suppress checks="JavadocType" files="[/\\]src[/\\]test[/\\]java[/\\]"/>
+
+ <!-- skip checks on customized titan 0.5.4 files -->
+ <suppress checks="[a-zA-Z0-9]*" files="[/\\]com[/\\]thinkaurelius[/\\]titan[/\\]"/>
</suppressions>
diff --git a/common/pom.xml b/common/pom.xml
index 6eb6638..6a57a3f 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -53,12 +53,17 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
diff --git a/distro/pom.xml b/distro/pom.xml
index 12a709c..799b154 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -32,7 +32,7 @@
<!-- by default configure hbase and solr with the distribution -->
<properties>
- <graph.storage.backend>hbase</graph.storage.backend>
+ <graph.storage.backend>hbase2</graph.storage.backend>
<graph.storage.properties>#Hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here
@@ -131,11 +131,12 @@ atlas.graph.index.search.solr.wait-searcher=true
<descriptor>src/main/assemblies/atlas-falcon-hook-package.xml</descriptor>
<descriptor>src/main/assemblies/atlas-sqoop-hook-package.xml</descriptor>
<descriptor>src/main/assemblies/atlas-storm-hook-package.xml</descriptor>
+ <descriptor>src/main/assemblies/atlas-falcon-hook-package.xml</descriptor>
<descriptor>src/main/assemblies/atlas-kafka-hook-package.xml</descriptor>
<descriptor>src/main/assemblies/atlas-server-package.xml</descriptor>
<descriptor>src/main/assemblies/standalone-package.xml</descriptor>
<descriptor>src/main/assemblies/src-package.xml</descriptor>
- <descriptor>src/main/assemblies/migration-exporter.xml</descriptor>
+ <!--<descriptor>src/main/assemblies/migration-exporter.xml</descriptor>-->
</descriptors>
<finalName>apache-atlas-${project.version}</finalName>
<tarLongFileMode>gnu</tarLongFileMode>
diff --git a/distro/src/bin/atlas_config.py b/distro/src/bin/atlas_config.py
index 747b03b..f09026f 100755
--- a/distro/src/bin/atlas_config.py
+++ b/distro/src/bin/atlas_config.py
@@ -32,7 +32,7 @@ LIB = "lib"
CONF = "conf"
LOG = "logs"
WEBAPP = "server" + os.sep + "webapp"
-CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "basic_configs" + os.sep + "conf"
+CONFIG_SETS_CONF = "server" + os.sep + "solr" + os.sep + "configsets" + os.sep + "_default" + os.sep + "conf"
DATA = "data"
ATLAS_CONF = "ATLAS_CONF"
ATLAS_LOG = "ATLAS_LOG_DIR"
@@ -63,7 +63,7 @@ ENV_KEYS = ["JAVA_HOME", ATLAS_OPTS, ATLAS_SERVER_OPTS, ATLAS_SERVER_HEAP, ATLAS
IS_WINDOWS = platform.system() == "Windows"
ON_POSIX = 'posix' in sys.builtin_module_names
CONF_FILE="atlas-application.properties"
-HBASE_STORAGE_CONF_ENTRY="atlas.graph.storage.backend\s*=\s*hbase"
+STORAGE_BACKEND_CONF="atlas.graph.storage.backend"
HBASE_STORAGE_LOCAL_CONF_ENTRY="atlas.graph.storage.hostname\s*=\s*localhost"
SOLR_INDEX_CONF_ENTRY="atlas.graph.index.search.backend\s*=\s*solr"
SOLR_INDEX_LOCAL_CONF_ENTRY="atlas.graph.index.search.solr.zookeeper-url\s*=\s*localhost"
@@ -405,15 +405,18 @@ def wait_for_shutdown(pid, msg, wait):
sys.stdout.write('\n')
def is_hbase(confdir):
- confdir = os.path.join(confdir, CONF_FILE)
- return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None
+ confFile = os.path.join(confdir, CONF_FILE)
+ storageBackEnd = getConfig(confFile, STORAGE_BACKEND_CONF)
+ if storageBackEnd is not None:
+ storageBackEnd = storageBackEnd.strip()
+ return storageBackEnd is None or storageBackEnd == '' or storageBackEnd == 'hbase' or storageBackEnd == 'hbase2'
def is_hbase_local(confdir):
if os.environ.get(MANAGE_LOCAL_HBASE, "False").lower() == 'false':
return False
- confdir = os.path.join(confdir, CONF_FILE)
- return grep(confdir, HBASE_STORAGE_CONF_ENTRY) is not None and grep(confdir, HBASE_STORAGE_LOCAL_CONF_ENTRY) is not None
+ confFile = os.path.join(confdir, CONF_FILE)
+ return is_hbase(confdir) and grep(confFile, HBASE_STORAGE_LOCAL_CONF_ENTRY) is not None
def run_hbase_action(dir, action, hbase_conf_dir = None, logdir = None, wait=True):
if IS_WINDOWS:
@@ -649,14 +652,14 @@ def configure_cassandra(dir):
def server_already_running(pid):
print "Atlas server is already running under process %s" % pid
- sys.exit()
-
+ sys.exit()
+
def server_pid_not_running(pid):
print "The Server is no longer running with pid %s" %pid
def grep(file, value):
for line in open(file).readlines():
- if re.match(value, line):
+ if re.match(value, line):
return line
return None
diff --git a/distro/src/conf/atlas-env.sh b/distro/src/conf/atlas-env.sh
index 298bc38..c4241e6 100644
--- a/distro/src/conf/atlas-env.sh
+++ b/distro/src/conf/atlas-env.sh
@@ -49,7 +49,7 @@
# Where pid files are stored. Defatult is logs directory under the base install location
#export ATLAS_PID_DIR=
-# where the atlas janusgraph db data is stored. Defatult is logs/data directory under the base install location
+# where the atlas titan db data is stored. Defatult is logs/data directory under the base install location
#export ATLAS_DATA_DIR=
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
diff --git a/distro/src/conf/solr/schema.xml b/distro/src/conf/solr/schema.xml
index bb90e63..1d445b1 100644
--- a/distro/src/conf/solr/schema.xml
+++ b/distro/src/conf/solr/schema.xml
@@ -519,6 +519,7 @@
-->
<fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
+ <!--Titan specific-->
<fieldType name="uuid"
class="solr.UUIDField"
indexed="true" />
diff --git a/distro/src/conf/solr/solrconfig.xml b/distro/src/conf/solr/solrconfig.xml
index 7664fd6..1d414f7 100644
--- a/distro/src/conf/solr/solrconfig.xml
+++ b/distro/src/conf/solr/solrconfig.xml
@@ -606,6 +606,7 @@
</admin>
+ <!--Titan specific-->
<updateRequestProcessorChain default="true">
<processor class="solr.TimestampUpdateProcessorFactory">
<str name="fieldName">timestamp</str>
diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml
index cba65c7..aa462cd 100755
--- a/distro/src/main/assemblies/standalone-package.xml
+++ b/distro/src/main/assemblies/standalone-package.xml
@@ -189,6 +189,21 @@
<directoryMode>0755</directoryMode>
</fileSet>
+ <!-- for migration setup -->
+ <fileSet>
+ <directory>../tools/atlas-migration-exporter</directory>
+ <outputDirectory>tools/migration-exporter</outputDirectory>
+ <includes>
+ <include>README</include>
+ <include>*.py</include>
+ <include>atlas-log4j.xml</include>
+ <include>atlas-migration-*.jar</include>
+ <include>migrationContext.xml</include>
+ </includes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
<fileSet>
<directory>../addons/kakfa-bridge/target/dependency/bridge</directory>
<outputDirectory>bridge</outputDirectory>
diff --git a/graphdb/janus-hbase2/pom.xml b/graphdb/janus-hbase2/pom.xml
new file mode 100644
index 0000000..1cb474c
--- /dev/null
+++ b/graphdb/janus-hbase2/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+~ Licensed to the Apache Software Foundation (ASF) under one
+~ or more contributor license agreements. See the NOTICE file
+~ distributed with this work for additional information
+~ regarding copyright ownership. The ASF licenses this file
+~ to you under the Apache License, Version 2.0 (the
+~ "License"); you may not use this file except in compliance
+~ with the License. You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>atlas-graphdb</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>2.0.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>atlas-janusgraph-hbase2</artifactId>
+ <description>Apache Atlas JanusGraph-HBase2 Module</description>
+ <name>Apache Atlas JanusGraph-HBase2 Module</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.janusgraph</groupId>
+ <artifactId>janusgraph-core</artifactId>
+ <version>${janus.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ <version>${hbase.version}</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <artifactId>avro</artifactId>
+ <groupId>org.apache.avro</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jruby-complete</artifactId>
+ <groupId>org.jruby</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>asm</artifactId>
+ <groupId>asm</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/AdminMask.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/AdminMask.java
new file mode 100644
index 0000000..548860b
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/AdminMask.java
@@ -0,0 +1,74 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Copyright DataStax, Inc.
+ * <p>
+ * Please see the included license file for details.
+ */
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its Admin/HBaseAdmin over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface AdminMask extends Closeable
+{
+
+ void clearTable(String tableName, long timestamp) throws IOException;
+
+ /**
+ * Drop given table. Table can be either enabled or disabled.
+ * @param tableName Name of the table to delete
+ * @throws IOException
+ */
+ void dropTable(String tableName) throws IOException;
+
+ TableDescriptor getTableDescriptor(String tableName) throws TableNotFoundException, IOException;
+
+ boolean tableExists(String tableName) throws IOException;
+
+ void createTable(TableDescriptor desc) throws IOException;
+
+ void createTable(TableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException;
+
+ /**
+ * Estimate the number of regionservers in the HBase cluster.
+ *
+ * This is usually implemented by calling
+ * {@link HBaseAdmin#getClusterStatus()} and then
+ * {@link ClusterStatus#getServers()} and finally {@code size()} on the
+ * returned server list.
+ *
+ * @return the number of servers in the cluster or -1 if it could not be determined
+ */
+ int getEstimatedRegionServerCount();
+
+ void disableTable(String tableName) throws IOException;
+
+ void enableTable(String tableName) throws IOException;
+
+ boolean isTableDisabled(String tableName) throws IOException;
+
+ void addColumn(String tableName, ColumnFamilyDescriptor columnDescriptor) throws IOException;
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/ConnectionMask.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/ConnectionMask.java
new file mode 100644
index 0000000..05ecd53
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/ConnectionMask.java
@@ -0,0 +1,55 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Copyright DataStax, Inc.
+ * <p>
+ * Please see the included license file for details.
+ */
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.HRegionLocation;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its (H)Connection class over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface ConnectionMask extends Closeable
+{
+
+ /**
+ * Retrieve the TableMask compatibility layer object for the supplied table name.
+ * @return The TableMask for the specified table.
+ * @throws IOException in the case of backend exceptions.
+ */
+ TableMask getTable(String name) throws IOException;
+
+ /**
+ * Retrieve the AdminMask compatibility layer object for this Connection.
+ * @return The AdminMask for this Connection
+ * @throws IOException in the case of backend exceptions.
+ */
+ AdminMask getAdmin() throws IOException;
+
+ /**
+ * Retrieve the RegionLocations for the supplied table name.
+ * @return A map of HRegionInfo to ServerName that describes the storage regions for the named table.
+ * @throws IOException in the case of backend exceptions.
+ */
+ List<HRegionLocation> getRegionLocations(String tablename) throws IOException;
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseAdmin2_0.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseAdmin2_0.java
new file mode 100644
index 0000000..f93481e
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseAdmin2_0.java
@@ -0,0 +1,167 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class HBaseAdmin2_0 implements AdminMask
+{
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseAdmin2_0.class);
+
+ private final Admin adm;
+
+ public HBaseAdmin2_0(Admin adm)
+ {
+ this.adm = adm;
+ }
+
+ /**
+ * Delete all rows from the given table. This method is intended only for development and testing use.
+ * @param tableString
+ * @param timestamp
+ * @throws IOException
+ */
+ @Override
+ public void clearTable(String tableString, long timestamp) throws IOException
+ {
+ TableName tableName = TableName.valueOf(tableString);
+
+ if (!adm.tableExists(tableName)) {
+ log.debug("Attempted to clear table {} before it exists (noop)", tableString);
+ return;
+ }
+
+ // Unfortunately, linear scanning and deleting rows is faster in HBase when running integration tests than
+ // disabling and deleting/truncating tables.
+ final Scan scan = new Scan();
+ scan.setCacheBlocks(false);
+ scan.setCaching(2000);
+ scan.setTimeRange(0, Long.MAX_VALUE);
+ scan.readVersions(1);
+
+ try (final Table table = adm.getConnection().getTable(tableName);
+ final ResultScanner scanner = table.getScanner(scan)) {
+ final Iterator<Result> iterator = scanner.iterator();
+ final int batchSize = 1000;
+ final List<Delete> deleteList = new ArrayList<>();
+ while (iterator.hasNext()) {
+ deleteList.add(new Delete(iterator.next().getRow(), timestamp));
+ if (!iterator.hasNext() || deleteList.size() == batchSize) {
+ table.delete(deleteList);
+ deleteList.clear();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void dropTable(String tableString) throws IOException {
+ final TableName tableName = TableName.valueOf(tableString);
+
+ if (!adm.tableExists(tableName)) {
+ log.debug("Attempted to drop table {} before it exists (noop)", tableString);
+ return;
+ }
+
+ if (adm.isTableEnabled(tableName)) {
+ adm.disableTable(tableName);
+ }
+ adm.deleteTable(tableName);
+ }
+
+ @Override
+ public TableDescriptor getTableDescriptor(String tableString) throws TableNotFoundException, IOException
+ {
+ return adm.getDescriptor(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public boolean tableExists(String tableString) throws IOException
+ {
+ return adm.tableExists(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void createTable(TableDescriptor desc) throws IOException
+ {
+ adm.createTable(desc);
+ }
+
+ @Override
+ public void createTable(TableDescriptor desc, byte[] startKey, byte[] endKey, int numRegions) throws IOException
+ {
+ adm.createTable(desc, startKey, endKey, numRegions);
+ }
+
+ @Override
+ public int getEstimatedRegionServerCount()
+ {
+ int serverCount = -1;
+ try {
+ serverCount = adm.getClusterStatus().getServers().size();
+ log.debug("Read {} servers from HBase ClusterStatus", serverCount);
+ } catch (IOException e) {
+ log.debug("Unable to retrieve HBase cluster status", e);
+ }
+ return serverCount;
+ }
+
+ @Override
+ public void disableTable(String tableString) throws IOException
+ {
+ adm.disableTable(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void enableTable(String tableString) throws IOException
+ {
+ adm.enableTable(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public boolean isTableDisabled(String tableString) throws IOException
+ {
+ return adm.isTableDisabled(TableName.valueOf(tableString));
+ }
+
+ @Override
+ public void addColumn(String tableString, ColumnFamilyDescriptor columnDescriptor) throws IOException
+ {
+ adm.addColumnFamily(TableName.valueOf(tableString), columnDescriptor);
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ adm.close();
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat.java
new file mode 100644
index 0000000..553ad46
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat.java
@@ -0,0 +1,58 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+
+import java.io.IOException;
+
+public interface HBaseCompat {
+
+ /**
+ * Configure the compression scheme {@code algo} on a column family
+ * descriptor {@code cd}. The {@code algo} parameter is a string value
+ * corresponding to one of the values of HBase's Compression enum. The
+ * Compression enum has moved between packages as HBase has evolved, which
+ * is why this method has a String argument in the signature instead of the
+ * enum itself.
+ * @param cd
+ * column family to configure
+ * @param algo
+ */
+ public ColumnFamilyDescriptor setCompression(ColumnFamilyDescriptor cd, String algo);
+
+ /**
+ * Create and return a HTableDescriptor instance with the given name. The
+ * constructors on this method have remained stable over HBase development
+ * so far, but the old HTableDescriptor(String) constructor & byte[] friends
+ * are now marked deprecated and may eventually be removed in favor of the
+ * HTableDescriptor(TableName) constructor. That constructor (and the
+ * TableName type) only exists in newer HBase versions. Hence this method.
+ *
+ * @param tableName
+ * HBase table name
+ * @return a new table descriptor instance
+ */
+ public TableDescriptor newTableDescriptor(String tableName);
+
+ ConnectionMask createConnection(Configuration conf) throws IOException;
+
+ TableDescriptor addColumnFamilyToTableDescriptor(TableDescriptor tdesc, ColumnFamilyDescriptor cdesc);
+
+ void setTimestamp(Delete d, long timestamp);
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat2_0.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat2_0.java
new file mode 100644
index 0000000..fdba24a
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompat2_0.java
@@ -0,0 +1,61 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.compress.Compression;
+
+import java.io.IOException;
+
+public class HBaseCompat2_0 implements HBaseCompat {
+
+ @Override
+ public ColumnFamilyDescriptor setCompression(ColumnFamilyDescriptor cd, String algo) {
+ return ColumnFamilyDescriptorBuilder.newBuilder(cd).setCompressionType(Compression.Algorithm.valueOf(algo)).build();
+ }
+
+ @Override
+ public TableDescriptor newTableDescriptor(String tableName) {
+ TableName tn = TableName.valueOf(tableName);
+
+ return TableDescriptorBuilder.newBuilder(tn).build();
+ }
+
+ @Override
+ public ConnectionMask createConnection(Configuration conf) throws IOException
+ {
+ return new HConnection2_0(ConnectionFactory.createConnection(conf));
+ }
+
+ @Override
+ public TableDescriptor addColumnFamilyToTableDescriptor(TableDescriptor tdesc, ColumnFamilyDescriptor cdesc)
+ {
+ return TableDescriptorBuilder.newBuilder(tdesc).addColumnFamily(cdesc).build();
+ }
+
+ @Override
+ public void setTimestamp(Delete d, long timestamp)
+ {
+ d.setTimestamp(timestamp);
+ }
+
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompatLoader.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompatLoader.java
new file mode 100644
index 0000000..d746b3d
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseCompatLoader.java
@@ -0,0 +1,90 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HBaseCompatLoader {
+
+ private static final Logger log = LoggerFactory.getLogger(HBaseCompatLoader.class);
+
+ private static final String DEFAULT_HBASE_COMPAT_VERSION = "1.2";
+
+ private static final String HBASE_VERSION_2_STRING = "2.";
+
+ private static final String DEFAULT_HBASE_COMPAT_CLASS_NAME =
+ "org.janusgraph.diskstorage.hbase2.HBaseCompat2_0";
+
+ private static final String[] HBASE_SUPPORTED_VERSIONS =
+ new String[] { "0.98", "1.0", "1.1", "1.2", "1.3", "2.0" };
+
+ private static HBaseCompat cachedCompat;
+
+ public synchronized static HBaseCompat getCompat(String classOverride) {
+
+ if (null != cachedCompat) {
+ log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
+ return cachedCompat;
+ }
+
+ HBaseCompat compat;
+ String className = null;
+ String classNameSource = null;
+
+ if (null != classOverride) {
+ className = classOverride;
+ classNameSource = "from explicit configuration";
+ } else {
+ String hbaseVersion = VersionInfo.getVersion();
+ for (String supportedVersion : HBASE_SUPPORTED_VERSIONS) {
+ if (hbaseVersion.startsWith(supportedVersion + ".")) {
+ if (hbaseVersion.startsWith(HBASE_VERSION_2_STRING)) {
+ // All HBase 2.x maps to HBaseCompat2_0.
+ className = DEFAULT_HBASE_COMPAT_CLASS_NAME;
+ }
+ else {
+ className = "org.janusgraph.diskstorage.hbase2.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
+ }
+ classNameSource = "supporting runtime HBase version " + hbaseVersion;
+ break;
+ }
+ }
+ if (null == className) {
+ log.info("The HBase version {} is not explicitly supported by JanusGraph. " +
+ "Loading JanusGraph's compatibility layer for its most recent supported HBase version ({})",
+ hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
+ className = DEFAULT_HBASE_COMPAT_CLASS_NAME;
+ classNameSource = " by default";
+ }
+ }
+
+ final String errTemplate = " when instantiating HBase compatibility class " + className;
+
+ try {
+ compat = (HBaseCompat)Class.forName(className).newInstance();
+ log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
+ }
+
+ return cachedCompat = compat;
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
new file mode 100644
index 0000000..9aa552d
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseKeyColumnValueStore.java
@@ -0,0 +1,384 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.ColumnPaginationFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.Entry;
+import org.janusgraph.diskstorage.EntryList;
+import org.janusgraph.diskstorage.EntryMetaData;
+import org.janusgraph.diskstorage.PermanentBackendException;
+import org.janusgraph.diskstorage.StaticBuffer;
+import org.janusgraph.diskstorage.TemporaryBackendException;
+import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
+import org.janusgraph.diskstorage.keycolumnvalue.KCVSUtil;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyIterator;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyRangeQuery;
+import org.janusgraph.diskstorage.keycolumnvalue.KeySliceQuery;
+import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
+import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
+import org.janusgraph.diskstorage.util.RecordIterator;
+import org.janusgraph.diskstorage.util.StaticArrayBuffer;
+import org.janusgraph.diskstorage.util.StaticArrayEntry;
+import org.janusgraph.diskstorage.util.StaticArrayEntryList;
+import org.janusgraph.util.system.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+/**
+ * Here are some areas that might need work:
+ * <p/>
+ * - batching? (consider HTable#batch, HTable#setAutoFlush(false)
+ * - tuning HTable#setWriteBufferSize (?)
+ * - writing a server-side filter to replace ColumnCountGetFilter, which drops
+ * all columns on the row where it reaches its limit. This requires getSlice,
+ * currently, to impose its limit on the client side. That obviously won't
+ * scale.
+ * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this)
+ * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache
+ * <p/>
+ * There may be other problem areas. These are just the ones of which I'm aware.
+ */
+public class HBaseKeyColumnValueStore implements KeyColumnValueStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class);
+
+ private final String tableName;
+ private final HBaseStoreManager storeManager;
+
+ // When using shortened CF names, columnFamily is the shortname and storeName is the longname
+ // When not using shortened CF names, they are the same
+ //private final String columnFamily;
+ private final String storeName;
+ // This is columnFamily.getBytes()
+ private final byte[] columnFamilyBytes;
+ private final HBaseGetter entryGetter;
+
+ private final ConnectionMask cnx;
+
+ HBaseKeyColumnValueStore(HBaseStoreManager storeManager, ConnectionMask cnx, String tableName, String columnFamily, String storeName) {
+ this.storeManager = storeManager;
+ this.cnx = cnx;
+ this.tableName = tableName;
+ //this.columnFamily = columnFamily;
+ this.storeName = storeName;
+ this.columnFamilyBytes = Bytes.toBytes(columnFamily);
+ this.entryGetter = new HBaseGetter(storeManager.getMetaDataSchema(storeName));
+ }
+
+ @Override
+ public void close() throws BackendException {
+ }
+
+ @Override
+ public EntryList getSlice(KeySliceQuery query, StoreTransaction txh) throws BackendException {
+ Map<StaticBuffer, EntryList> result = getHelper(Arrays.asList(query.getKey()), getFilter(query));
+ return Iterables.getOnlyElement(result.values(), EntryList.EMPTY_LIST);
+ }
+
+ @Override
+ public Map<StaticBuffer,EntryList> getSlice(List<StaticBuffer> keys, SliceQuery query, StoreTransaction txh) throws BackendException {
+ return getHelper(keys, getFilter(query));
+ }
+
+ @Override
+ public void mutate(StaticBuffer key, List<Entry> additions, List<StaticBuffer> deletions, StoreTransaction txh) throws BackendException {
+ Map<StaticBuffer, KCVMutation> mutations = ImmutableMap.of(key, new KCVMutation(additions, deletions));
+ mutateMany(mutations, txh);
+ }
+
+ @Override
+ public void acquireLock(StaticBuffer key,
+ StaticBuffer column,
+ StaticBuffer expectedValue,
+ StoreTransaction txh) throws BackendException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws BackendException {
+ return executeKeySliceQuery(query.getKeyStart().as(StaticBuffer.ARRAY_FACTORY),
+ query.getKeyEnd().as(StaticBuffer.ARRAY_FACTORY),
+ new FilterList(FilterList.Operator.MUST_PASS_ALL),
+ query);
+ }
+
+ @Override
+ public String getName() {
+ return storeName;
+ }
+
+ @Override
+ public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws BackendException {
+ return executeKeySliceQuery(new FilterList(FilterList.Operator.MUST_PASS_ALL), query);
+ }
+
+ public static Filter getFilter(SliceQuery query) {
+ byte[] colStartBytes = query.getSliceStart().length() > 0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null;
+ byte[] colEndBytes = query.getSliceEnd().length() > 0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null;
+
+ Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false);
+
+ if (query.hasLimit()) {
+ filter = new FilterList(FilterList.Operator.MUST_PASS_ALL,
+ filter,
+ new ColumnPaginationFilter(query.getLimit(), 0));
+ }
+
+ logger.debug("Generated HBase Filter {}", filter);
+
+ return filter;
+ }
+
+ private Map<StaticBuffer,EntryList> getHelper(List<StaticBuffer> keys, Filter getFilter) throws BackendException {
+ List<Get> requests = new ArrayList<Get>(keys.size());
+ {
+ for (StaticBuffer key : keys) {
+ Get g = new Get(key.as(StaticBuffer.ARRAY_FACTORY)).addFamily(columnFamilyBytes).setFilter(getFilter);
+ try {
+ g.setTimeRange(0, Long.MAX_VALUE);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+ requests.add(g);
+ }
+ }
+
+ Map<StaticBuffer,EntryList> resultMap = new HashMap<StaticBuffer,EntryList>(keys.size());
+
+ try {
+ TableMask table = null;
+ Result[] results = null;
+
+ try {
+ table = cnx.getTable(tableName);
+ results = table.get(requests);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+
+ if (results == null)
+ return KCVSUtil.emptyResults(keys);
+
+ assert results.length==keys.size();
+
+ for (int i = 0; i < results.length; i++) {
+ Result result = results[i];
+ NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> f = result.getMap();
+
+ if (f == null) { // no result for this key
+ resultMap.put(keys.get(i), EntryList.EMPTY_LIST);
+ continue;
+ }
+
+ // actual key with <timestamp, value>
+ NavigableMap<byte[], NavigableMap<Long, byte[]>> r = f.get(columnFamilyBytes);
+ resultMap.put(keys.get(i), (r == null)
+ ? EntryList.EMPTY_LIST
+ : StaticArrayEntryList.ofBytes(r.entrySet(), entryGetter));
+ }
+
+ return resultMap;
+ } catch (InterruptedIOException e) {
+ // added to support traversal interruption
+ Thread.currentThread().interrupt();
+ throw new PermanentBackendException(e);
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ private void mutateMany(Map<StaticBuffer, KCVMutation> mutations, StoreTransaction txh) throws BackendException {
+ storeManager.mutateMany(ImmutableMap.of(storeName, mutations), txh);
+ }
+
+ private KeyIterator executeKeySliceQuery(FilterList filters, @Nullable SliceQuery columnSlice) throws BackendException {
+ return executeKeySliceQuery(null, null, filters, columnSlice);
+ }
+
+ private KeyIterator executeKeySliceQuery(@Nullable byte[] startKey,
+ @Nullable byte[] endKey,
+ FilterList filters,
+ @Nullable SliceQuery columnSlice) throws BackendException {
+ Scan scan = new Scan().addFamily(columnFamilyBytes);
+
+ try {
+ scan.setTimeRange(0, Long.MAX_VALUE);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+
+ if (startKey != null)
+ scan.withStartRow(startKey);
+
+ if (endKey != null)
+ scan.withStopRow(endKey);
+
+ if (columnSlice != null) {
+ filters.addFilter(getFilter(columnSlice));
+ }
+
+ TableMask table = null;
+
+ try {
+ table = cnx.getTable(tableName);
+ return new RowIterator(table, table.getScanner(scan.setFilter(filters)), columnFamilyBytes);
+ } catch (IOException e) {
+ IOUtils.closeQuietly(table);
+ throw new PermanentBackendException(e);
+ }
+ }
+
+ private class RowIterator implements KeyIterator {
+ private final Closeable table;
+ private final Iterator<Result> rows;
+ private final byte[] columnFamilyBytes;
+
+ private Result currentRow;
+ private boolean isClosed;
+
+ public RowIterator(Closeable table, ResultScanner rows, byte[] columnFamilyBytes) {
+ this.table = table;
+ this.columnFamilyBytes = Arrays.copyOf(columnFamilyBytes, columnFamilyBytes.length);
+ this.rows = Iterators.filter(rows.iterator(), result -> null != result && null != result.getRow());
+ }
+
+ @Override
+ public RecordIterator<Entry> getEntries() {
+ ensureOpen();
+
+ return new RecordIterator<Entry>() {
+ private final Iterator<Map.Entry<byte[], NavigableMap<Long, byte[]>>> kv;
+ {
+ final Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = currentRow.getMap();
+ Preconditions.checkNotNull(map);
+ kv = map.get(columnFamilyBytes).entrySet().iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ ensureOpen();
+ return kv.hasNext();
+ }
+
+ @Override
+ public Entry next() {
+ ensureOpen();
+ return StaticArrayEntry.ofBytes(kv.next(), entryGetter);
+ }
+
+ @Override
+ public void close() {
+ isClosed = true;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public boolean hasNext() {
+ ensureOpen();
+ return rows.hasNext();
+ }
+
+ @Override
+ public StaticBuffer next() {
+ ensureOpen();
+
+ currentRow = rows.next();
+ return StaticArrayBuffer.of(currentRow.getRow());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(table);
+ isClosed = true;
+ logger.debug("RowIterator closed table {}", table);
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void ensureOpen() {
+ if (isClosed)
+ throw new IllegalStateException("Iterator has been closed.");
+ }
+ }
+
+ private static class HBaseGetter implements StaticArrayEntry.GetColVal<Map.Entry<byte[], NavigableMap<Long, byte[]>>, byte[]> {
+
+ private final EntryMetaData[] schema;
+
+ private HBaseGetter(EntryMetaData[] schema) {
+ this.schema = schema;
+ }
+
+ @Override
+ public byte[] getColumn(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return element.getKey();
+ }
+
+ @Override
+ public byte[] getValue(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return element.getValue().lastEntry().getValue();
+ }
+
+ @Override
+ public EntryMetaData[] getMetaSchema(Map.Entry<byte[], NavigableMap<Long, byte[]>> element) {
+ return schema;
+ }
+
+ @Override
+ public Object getMetaData(Map.Entry<byte[], NavigableMap<Long, byte[]>> element, EntryMetaData meta) {
+ switch(meta) {
+ case TIMESTAMP:
+ return element.getValue().lastEntry().getKey();
+ default:
+ throw new UnsupportedOperationException("Unsupported meta data: " + meta);
+ }
+ }
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
new file mode 100644
index 0000000..f857012
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseStoreManager.java
@@ -0,0 +1,986 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.janusgraph.core.JanusGraphException;
+import org.janusgraph.diskstorage.BackendException;
+import org.janusgraph.diskstorage.BaseTransactionConfig;
+import org.janusgraph.diskstorage.Entry;
+import org.janusgraph.diskstorage.EntryMetaData;
+import org.janusgraph.diskstorage.PermanentBackendException;
+import org.janusgraph.diskstorage.StaticBuffer;
+import org.janusgraph.diskstorage.StoreMetaData;
+import org.janusgraph.diskstorage.TemporaryBackendException;
+import org.janusgraph.diskstorage.common.DistributedStoreManager;
+import org.janusgraph.diskstorage.configuration.ConfigElement;
+import org.janusgraph.diskstorage.configuration.ConfigNamespace;
+import org.janusgraph.diskstorage.configuration.ConfigOption;
+import org.janusgraph.diskstorage.configuration.Configuration;
+import org.janusgraph.diskstorage.keycolumnvalue.KCVMutation;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStore;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager;
+import org.janusgraph.diskstorage.keycolumnvalue.KeyRange;
+import org.janusgraph.diskstorage.keycolumnvalue.StandardStoreFeatures;
+import org.janusgraph.diskstorage.keycolumnvalue.StoreFeatures;
+import org.janusgraph.diskstorage.keycolumnvalue.StoreTransaction;
+import org.janusgraph.diskstorage.util.BufferUtil;
+import org.janusgraph.diskstorage.util.StaticArrayBuffer;
+import org.janusgraph.diskstorage.util.time.TimestampProviders;
+import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
+import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions;
+import org.janusgraph.util.system.IOUtils;
+import org.janusgraph.util.system.NetworkUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.janusgraph.diskstorage.Backend.EDGESTORE_NAME;
+import static org.janusgraph.diskstorage.Backend.INDEXSTORE_NAME;
+import static org.janusgraph.diskstorage.Backend.LOCK_STORE_SUFFIX;
+import static org.janusgraph.diskstorage.Backend.SYSTEM_MGMT_LOG_NAME;
+import static org.janusgraph.diskstorage.Backend.SYSTEM_TX_LOG_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DROP_ON_CLEAR;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.GRAPH_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.IDS_STORE_NAME;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME;
+
+/**
+ * Storage Manager for HBase
+ */
+@PreInitializeConfigOptions
+public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class);
+
+ public static final ConfigNamespace HBASE_NS =
+ new ConfigNamespace(GraphDatabaseConfiguration.STORAGE_NS, "hbase", "HBase storage options");
+
+ public static final ConfigOption<Boolean> SHORT_CF_NAMES =
+ new ConfigOption<>(HBASE_NS, "short-cf-names",
+ "Whether to shorten the names of JanusGraph's column families to one-character mnemonics " +
+ "to conserve storage space", ConfigOption.Type.FIXED, true);
+
+ public static final String COMPRESSION_DEFAULT = "-DEFAULT-";
+
+ public static final ConfigOption<String> COMPRESSION =
+ new ConfigOption<>(HBASE_NS, "compression-algorithm",
+ "An HBase Compression.Algorithm enum string which will be applied to newly created column families. " +
+ "The compression algorithm must be installed and available on the HBase cluster. JanusGraph cannot install " +
+ "and configure new compression algorithms on the HBase cluster by itself.",
+ ConfigOption.Type.MASKABLE, "GZ");
+
+ public static final ConfigOption<Boolean> SKIP_SCHEMA_CHECK =
+ new ConfigOption<>(HBASE_NS, "skip-schema-check",
+ "Assume that JanusGraph's HBase table and column families already exist. " +
+ "When this is true, JanusGraph will not check for the existence of its table/CFs, " +
+ "nor will it attempt to create them under any circumstances. This is useful " +
+ "when running JanusGraph without HBase admin privileges.",
+ ConfigOption.Type.MASKABLE, false);
+
+ public static final ConfigOption<String> HBASE_TABLE =
+ new ConfigOption<>(HBASE_NS, "table",
+ "The name of the table JanusGraph will use. When " + ConfigElement.getPath(SKIP_SCHEMA_CHECK) +
+ " is false, JanusGraph will automatically create this table if it does not already exist." +
+ " If this configuration option is not provided but graph.graphname is, the table will be set" +
+ " to that value.",
+ ConfigOption.Type.LOCAL, "janusgraph");
+
+ /**
+ * Related bug fixed in 0.98.0, 0.94.7, 0.95.0:
+ *
+ * https://issues.apache.org/jira/browse/HBASE-8170
+ */
+ public static final int MIN_REGION_COUNT = 3;
+
+ /**
+ * The total number of HBase regions to create with JanusGraph's table. This
+ * setting only effects table creation; this normally happens just once when
+ * JanusGraph connects to an HBase backend for the first time.
+ */
+ public static final ConfigOption<Integer> REGION_COUNT =
+ new ConfigOption<Integer>(HBASE_NS, "region-count",
+ "The number of initial regions set when creating JanusGraph's HBase table",
+ ConfigOption.Type.MASKABLE, Integer.class, input -> null != input && MIN_REGION_COUNT <= input);
+
+ /**
+ * This setting is used only when {@link #REGION_COUNT} is unset.
+ * <p/>
+ * If JanusGraph's HBase table does not exist, then it will be created with total
+ * region count = (number of servers reported by ClusterStatus) * (this
+ * value).
+ * <p/>
+ * The Apache HBase manual suggests an order-of-magnitude range of potential
+ * values for this setting:
+ *
+ * <ul>
+ * <li>
+ * <a href="https://hbase.apache.org/book/important_configurations.html#disable.splitting">2.5.2.7. Managed Splitting</a>:
+ * <blockquote>
+ * What's the optimal number of pre-split regions to create? Mileage will
+ * vary depending upon your application. You could start low with 10
+ * pre-split regions / server and watch as data grows over time. It's
+ * better to err on the side of too little regions and rolling split later.
+ * </blockquote>
+ * </li>
+ * <li>
+ * <a href="https://hbase.apache.org/book/regions.arch.html">9.7 Regions</a>:
+ * <blockquote>
+ * In general, HBase is designed to run with a small (20-200) number of
+ * relatively large (5-20Gb) regions per server... Typically you want to
+ * keep your region count low on HBase for numerous reasons. Usually
+ * right around 100 regions per RegionServer has yielded the best results.
+ * </blockquote>
+ * </li>
+ * </ul>
+ *
+ * These considerations may differ for other HBase implementations (e.g. MapR).
+ */
+ public static final ConfigOption<Integer> REGIONS_PER_SERVER =
+ new ConfigOption<>(HBASE_NS, "regions-per-server",
+ "The number of regions per regionserver to set when creating JanusGraph's HBase table",
+ ConfigOption.Type.MASKABLE, Integer.class);
+
+ /**
+ * If this key is present in either the JVM system properties or the process
+ * environment (checked in the listed order, first hit wins), then its value
+ * must be the full package and class name of an implementation of
+ * {@link HBaseCompat} that has a no-arg public constructor.
+ * <p>
+ * When this <b>is not</b> set, JanusGraph attempts to automatically detect the
+ * HBase runtime version by calling {@link VersionInfo#getVersion()}. JanusGraph
+ * then checks the returned version string against a hard-coded list of
+ * supported version prefixes and instantiates the associated compat layer
+ * if a match is found.
+ * <p>
+ * When this <b>is</b> set, JanusGraph will not call
+ * {@code VersionInfo.getVersion()} or read its hard-coded list of supported
+ * version prefixes. JanusGraph will instead attempt to instantiate the class
+ * specified (via the no-arg constructor which must exist) and then attempt
+ * to cast it to HBaseCompat and use it as such. JanusGraph will assume the
+ * supplied implementation is compatible with the runtime HBase version and
+ * make no attempt to verify that assumption.
+ * <p>
+ * Setting this key incorrectly could cause runtime exceptions at best or
+ * silent data corruption at worst. This setting is intended for users
+ * running exotic HBase implementations that don't support VersionInfo or
+ * implementations which return values from {@code VersionInfo.getVersion()}
+ * that are inconsistent with Apache's versioning convention. It may also be
+ * useful to users who want to run against a new release of HBase that JanusGraph
+ * doesn't yet officially support.
+ *
+ */
+ public static final ConfigOption<String> COMPAT_CLASS =
+ new ConfigOption<>(HBASE_NS, "compat-class",
+ "The package and class name of the HBaseCompat implementation. HBaseCompat masks version-specific HBase API differences. " +
+ "When this option is unset, JanusGraph calls HBase's VersionInfo.getVersion() and loads the matching compat class " +
+ "at runtime. Setting this option forces JanusGraph to instead reflectively load and instantiate the specified class.",
+ ConfigOption.Type.MASKABLE, String.class);
+
+ public static final int PORT_DEFAULT = 9160;
+
+ public static final TimestampProviders PREFERRED_TIMESTAMPS = TimestampProviders.MILLI;
+
+ public static final ConfigNamespace HBASE_CONFIGURATION_NAMESPACE =
+ new ConfigNamespace(HBASE_NS, "ext", "Overrides for hbase-{site,default}.xml options", true);
+
+ private static final StaticBuffer FOUR_ZERO_BYTES = BufferUtil.zeroBuffer(4);
+
+ // Immutable instance fields
+ private final BiMap<String, String> shortCfNameMap;
+ private final String tableName;
+ private final String compression;
+ private final int regionCount;
+ private final int regionsPerServer;
+ private final ConnectionMask cnx;
+ private final org.apache.hadoop.conf.Configuration hconf;
+ private final boolean shortCfNames;
+ private final boolean skipSchemaCheck;
+ private final String compatClass;
+ private final HBaseCompat compat;
+ // Cached return value of getDeployment() as requesting it can be expensive.
+ private Deployment deployment = null;
+
+ private static final ConcurrentHashMap<HBaseStoreManager, Throwable> openManagers = new ConcurrentHashMap<>();
+
+ // Mutable instance state
+ private final ConcurrentMap<String, HBaseKeyColumnValueStore> openStores;
+
+ public HBaseStoreManager(org.janusgraph.diskstorage.configuration.Configuration config) throws BackendException {
+ super(config, PORT_DEFAULT);
+
+ shortCfNameMap = createShortCfMap(config);
+
+ Preconditions.checkArgument(null != shortCfNameMap);
+ Collection<String> shorts = shortCfNameMap.values();
+ Preconditions.checkArgument(Sets.newHashSet(shorts).size() == shorts.size());
+
+ checkConfigDeprecation(config);
+
+ this.tableName = determineTableName(config);
+ this.compression = config.get(COMPRESSION);
+ this.regionCount = config.has(REGION_COUNT) ? config.get(REGION_COUNT) : -1;
+ this.regionsPerServer = config.has(REGIONS_PER_SERVER) ? config.get(REGIONS_PER_SERVER) : -1;
+ this.skipSchemaCheck = config.get(SKIP_SCHEMA_CHECK);
+ this.compatClass = config.has(COMPAT_CLASS) ? config.get(COMPAT_CLASS) : null;
+ this.compat = HBaseCompatLoader.getCompat(compatClass);
+
+ /*
+ * Specifying both region count options is permitted but may be
+ * indicative of a misunderstanding, so issue a warning.
+ */
+ if (config.has(REGIONS_PER_SERVER) && config.has(REGION_COUNT)) {
+ logger.warn("Both {} and {} are set in JanusGraph's configuration, but "
+ + "the former takes precedence and the latter will be ignored.",
+ REGION_COUNT, REGIONS_PER_SERVER);
+ }
+
+ /* This static factory calls HBaseConfiguration.addHbaseResources(),
+ * which in turn applies the contents of hbase-default.xml and then
+ * applies the contents of hbase-site.xml.
+ */
+ this.hconf = HBaseConfiguration.create();
+
+ // Copy a subset of our commons config into a Hadoop config
+ int keysLoaded=0;
+ Map<String,Object> configSub = config.getSubset(HBASE_CONFIGURATION_NAMESPACE);
+ for (Map.Entry<String,Object> entry : configSub.entrySet()) {
+ logger.info("HBase configuration: setting {}={}", entry.getKey(), entry.getValue());
+ if (entry.getValue()==null) continue;
+ hconf.set(entry.getKey(), entry.getValue().toString());
+ keysLoaded++;
+ }
+
+ // Special case for STORAGE_HOSTS
+ if (config.has(GraphDatabaseConfiguration.STORAGE_HOSTS)) {
+ String zkQuorumKey = "hbase.zookeeper.quorum";
+ String csHostList = Joiner.on(",").join(config.get(GraphDatabaseConfiguration.STORAGE_HOSTS));
+ hconf.set(zkQuorumKey, csHostList);
+ logger.info("Copied host list from {} to {}: {}", GraphDatabaseConfiguration.STORAGE_HOSTS, zkQuorumKey, csHostList);
+ }
+
+ logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
+
+ this.shortCfNames = config.get(SHORT_CF_NAMES);
+
+ try {
+ //this.cnx = HConnectionManager.createConnection(hconf);
+ this.cnx = compat.createConnection(hconf);
+ } catch (IOException e) {
+ throw new PermanentBackendException(e);
+ }
+
+ if (logger.isTraceEnabled()) {
+ openManagers.put(this, new Throwable("Manager Opened"));
+ dumpOpenManagers();
+ }
+
+ logger.debug("Dumping HBase config key=value pairs");
+ for (Map.Entry<String, String> entry : hconf) {
+ logger.debug("[HBaseConfig] " + entry.getKey() + "=" + entry.getValue());
+ }
+ logger.debug("End of HBase config key=value pairs");
+
+ openStores = new ConcurrentHashMap<>();
+ }
+
+ public static BiMap<String, String> createShortCfMap(Configuration config) {
+ return ImmutableBiMap.<String, String>builder()
+ .put(INDEXSTORE_NAME, "g")
+ .put(INDEXSTORE_NAME + LOCK_STORE_SUFFIX, "h")
+ .put(config.get(IDS_STORE_NAME), "i")
+ .put(EDGESTORE_NAME, "e")
+ .put(EDGESTORE_NAME + LOCK_STORE_SUFFIX, "f")
+ .put(SYSTEM_PROPERTIES_STORE_NAME, "s")
+ .put(SYSTEM_PROPERTIES_STORE_NAME + LOCK_STORE_SUFFIX, "t")
+ .put(SYSTEM_MGMT_LOG_NAME, "m")
+ .put(SYSTEM_TX_LOG_NAME, "l")
+ .build();
+ }
+
+ @Override
+ public Deployment getDeployment() {
+ if (null != deployment) {
+ return deployment;
+ }
+
+ List<KeyRange> local;
+ try {
+ local = getLocalKeyPartition();
+ deployment = null != local && !local.isEmpty() ? Deployment.LOCAL : Deployment.REMOTE;
+ } catch (BackendException e) {
+ throw new RuntimeException(e);
+ }
+ return deployment;
+ }
+
+ @Override
+ public String toString() {
+ return "hbase[" + tableName + "@" + super.toString() + "]";
+ }
+
+ public void dumpOpenManagers() {
+ int estimatedSize = openManagers.size();
+ logger.trace("---- Begin open HBase store manager list ({} managers) ----", estimatedSize);
+ for (HBaseStoreManager m : openManagers.keySet()) {
+ logger.trace("Manager {} opened at:", m, openManagers.get(m));
+ }
+ logger.trace("---- End open HBase store manager list ({} managers) ----", estimatedSize);
+ }
+
+ @Override
+ public void close() {
+ openStores.clear();
+ if (logger.isTraceEnabled())
+ openManagers.remove(this);
+ IOUtils.closeQuietly(cnx);
+ }
+
+ @Override
+ public StoreFeatures getFeatures() {
+
+ Configuration c = GraphDatabaseConfiguration.buildGraphConfiguration();
+
+ StandardStoreFeatures.Builder fb = new StandardStoreFeatures.Builder()
+ .orderedScan(true).unorderedScan(true).batchMutation(true)
+ .multiQuery(true).distributed(true).keyOrdered(true).storeTTL(true)
+ .cellTTL(true).timestamps(true).preferredTimestamps(PREFERRED_TIMESTAMPS)
+ .optimisticLocking(true).keyConsistent(c);
+
+ try {
+ fb.localKeyPartition(getDeployment() == Deployment.LOCAL);
+ } catch (Exception e) {
+ logger.warn("Unexpected exception during getDeployment()", e);
+ }
+
+ return fb.build();
+ }
+
+ @Override
+ public void mutateMany(Map<String, Map<StaticBuffer, KCVMutation>> mutations, StoreTransaction txh) throws BackendException {
+ final MaskedTimestamp commitTime = new MaskedTimestamp(txh);
+ // In case of an addition and deletion with identical timestamps, the
+ // deletion tombstone wins.
+ // http://hbase.apache.org/book/versions.html#d244e4250
+ final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey =
+ convertToCommands(
+ mutations,
+ commitTime.getAdditionTime(times),
+ commitTime.getDeletionTime(times));
+
+ final List<Row> batch = new ArrayList<>(commandsPerKey.size()); // actual batch operation
+
+ // convert sorted commands into representation required for 'batch' operation
+ for (Pair<List<Put>, Delete> commands : commandsPerKey.values()) {
+ if (commands.getFirst() != null && !commands.getFirst().isEmpty())
+ batch.addAll(commands.getFirst());
+
+ if (commands.getSecond() != null)
+ batch.add(commands.getSecond());
+ }
+
+ try {
+ TableMask table = null;
+
+ try {
+ table = cnx.getTable(tableName);
+ table.batch(batch, new Object[batch.size()]);
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ } catch (InterruptedException e) {
+ throw new TemporaryBackendException(e);
+ }
+
+ sleepAfterWrite(txh, commitTime);
+ }
+
+ @Override
+ public KeyColumnValueStore openDatabase(String longName, StoreMetaData.Container metaData) throws BackendException {
+ // HBase does not support retrieving cell-level TTL by the client.
+ Preconditions.checkArgument(!storageConfig.has(GraphDatabaseConfiguration.STORE_META_TTL, longName)
+ || !storageConfig.get(GraphDatabaseConfiguration.STORE_META_TTL, longName));
+
+ HBaseKeyColumnValueStore store = openStores.get(longName);
+
+ if (store == null) {
+ final String cfName = getCfNameForStoreName(longName);
+
+ HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(this, cnx, tableName, cfName, longName);
+
+ store = openStores.putIfAbsent(longName, newStore); // nothing bad happens if we loose to other thread
+
+ if (store == null) {
+ if (!skipSchemaCheck) {
+ int cfTTLInSeconds = -1;
+ if (metaData.contains(StoreMetaData.TTL)) {
+ cfTTLInSeconds = metaData.get(StoreMetaData.TTL);
+ }
+ ensureColumnFamilyExists(tableName, cfName, cfTTLInSeconds);
+ }
+
+ store = newStore;
+ }
+ }
+
+ return store;
+ }
+
+ @Override
+ public StoreTransaction beginTransaction(final BaseTransactionConfig config) throws BackendException {
+ return new HBaseTransaction(config);
+ }
+
+ @Override
+ public String getName() {
+ return tableName;
+ }
+
+ /**
+ * Deletes the specified table with all its columns.
+ * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss.
+ */
+ @Override
+ public void clearStorage() throws BackendException {
+ try (AdminMask adm = getAdminInterface()) {
+ if (this.storageConfig.get(DROP_ON_CLEAR)) {
+ adm.dropTable(tableName);
+ } else {
+ adm.clearTable(tableName, times.getTime(times.getTime()));
+ }
+ } catch (IOException e)
+ {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ @Override
+ public boolean exists() throws BackendException {
+ try (final AdminMask adm = getAdminInterface()) {
+ return adm.tableExists(tableName);
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+ }
+
+ @Override
+ public List<KeyRange> getLocalKeyPartition() throws BackendException {
+ List<KeyRange> result = new LinkedList<>();
+ try {
+ ensureTableExists(
+ tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
+ Map<KeyRange, ServerName> normed = normalizeKeyBounds(cnx.getRegionLocations(tableName));
+
+ for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
+ if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
+ result.add(e.getKey());
+ logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
+ } else {
+ logger.debug("Discarding remote {}", e.getValue());
+ }
+ }
+ } catch (MasterNotRunningException e) {
+ logger.warn("Unexpected MasterNotRunningException", e);
+ } catch (ZooKeeperConnectionException e) {
+ logger.warn("Unexpected ZooKeeperConnectionException", e);
+ } catch (IOException e) {
+ logger.warn("Unexpected IOException", e);
+ }
+ return result;
+ }
+
+ /**
+ * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
+ * region's start and end key bounds using JanusGraph-partitioning-friendly
+ * conventions (start inclusive, end exclusive, zero bytes appended where
+ * necessary to make all keys at least 4 bytes long).
+ * <p/>
+ * This method iterates over the entries in its map parameter and performs
+ * the following conditional conversions on its keys. "Require" below means
+ * either a {@link Preconditions} invocation or an assertion. HRegionInfo
+ * sometimes returns start and end keys of zero length; this method replaces
+ * zero length keys with null before doing any of the checks described
+ * below. The parameter map and the values it contains are only read and
+ * never modified.
+ *
+ * <ul>
+ * <li>If an entry's HRegionInfo has null start and end keys, then first
+ * require that the parameter map is a singleton, and then return a
+ * single-entry map whose {@code KeyRange} has start and end buffers that
+ * are both four bytes of zeros.</li>
+ * <li>If the entry has a null end key (but non-null start key), put an
+ * equivalent entry in the result map with a start key identical to the
+ * input, except that zeros are appended to values less than 4 bytes long,
+ * and an end key that is four bytes of zeros.
+ * <li>If the entry has a null start key (but non-null end key), put an
+ * equivalent entry in the result map where the start key is four bytes of
+ * zeros, and the end key has zeros appended, if necessary, to make it at
+ * least 4 bytes long, after which one is added to the padded value in
+ * unsigned 32-bit arithmetic with overflow allowed.</li>
+ * <li>Any entry which matches none of the above criteria results in an
+ * equivalent entry in the returned map, except that zeros are appended to
+ * both keys to make each at least 4 bytes long, and the end key is then
+ * incremented as described in the last bullet point.</li>
+ * </ul>
+ *
+ * After iterating over the parameter map, this method checks that it either
+ * saw no entries with null keys, one entry with a null start key and a
+ * different entry with a null end key, or one entry with both start and end
+ * keys null. If any null keys are observed besides these three cases, the
+ * method will die with a precondition failure.
+ *
+ * @param locations A list of HRegionInfo
+ * @return JanusGraph-friendly expression of each region's rowkey boundaries
+ */
+ private Map<KeyRange, ServerName> normalizeKeyBounds(List<HRegionLocation> locations) {
+
+ HRegionLocation nullStart = null;
+ HRegionLocation nullEnd = null;
+
+ ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
+
+ for (HRegionLocation location : locations) {
+ HRegionInfo regionInfo = location.getRegionInfo();
+ ServerName serverName = location.getServerName();
+ byte startKey[] = regionInfo.getStartKey();
+ byte endKey[] = regionInfo.getEndKey();
+
+ if (0 == startKey.length) {
+ startKey = null;
+ logger.trace("Converted zero-length HBase startKey byte array to null");
+ }
+
+ if (0 == endKey.length) {
+ endKey = null;
+ logger.trace("Converted zero-length HBase endKey byte array to null");
+ }
+
+ if (null == startKey && null == endKey) {
+ Preconditions.checkState(1 == locations.size());
+ logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
+ // Choose arbitrary shared value = startKey = endKey
+ return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), serverName).build();
+ } else if (null == startKey) {
+ logger.debug("Found HRegionInfo with null startKey on server {}: {}", serverName, regionInfo);
+ Preconditions.checkState(null == nullStart);
+ nullStart = location;
+ // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
+ StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
+ // Replace null start key with zeroes
+ b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), serverName);
+ } else if (null == endKey) {
+ logger.debug("Found HRegionInfo with null endKey on server {}: {}", serverName, regionInfo);
+ Preconditions.checkState(null == nullEnd);
+ nullEnd = location;
+ // Replace null end key with zeroes
+ b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), serverName);
+ } else {
+ Preconditions.checkState(null != startKey);
+ Preconditions.checkState(null != endKey);
+
+ // Convert HBase's inclusive end keys into exclusive JanusGraph end keys
+ StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
+ StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
+
+ KeyRange kr = new KeyRange(startBuf, endBuf);
+ b.put(kr, serverName);
+ logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", serverName, regionInfo);
+ }
+ }
+
+ // Require either no null key bounds or a pair of them
+ Preconditions.checkState(!(null == nullStart ^ null == nullEnd));
+
+ // Check that every key in the result is at least 4 bytes long
+ Map<KeyRange, ServerName> result = b.build();
+ for (KeyRange kr : result.keySet()) {
+ Preconditions.checkState(4 <= kr.getStart().length());
+ Preconditions.checkState(4 <= kr.getEnd().length());
+ }
+
+ return result;
+ }
+
+ /**
+ * If the parameter is shorter than 4 bytes, then create and return a new 4
+ * byte array with the input array's bytes followed by zero bytes. Otherwise
+ * return the parameter.
+ *
+ * @param dataToPad non-null but possibly zero-length byte array
+ * @return either the parameter or a new array
+ */
+ private final byte[] zeroExtend(byte[] dataToPad) {
+ assert null != dataToPad;
+
+ final int targetLength = 4;
+
+ if (targetLength <= dataToPad.length)
+ return dataToPad;
+
+ byte padded[] = new byte[targetLength];
+
+ for (int i = 0; i < dataToPad.length; i++)
+ padded[i] = dataToPad[i];
+
+ for (int i = dataToPad.length; i < padded.length; i++)
+ padded[i] = (byte)0;
+
+ return padded;
+ }
+
+ public static String shortenCfName(BiMap<String, String> shortCfNameMap, String longName) throws PermanentBackendException {
+ final String s;
+ if (shortCfNameMap.containsKey(longName)) {
+ s = shortCfNameMap.get(longName);
+ Preconditions.checkNotNull(s);
+ logger.debug("Substituted default CF name \"{}\" with short form \"{}\" to reduce HBase KeyValue size", longName, s);
+ } else {
+ if (shortCfNameMap.containsValue(longName)) {
+ String fmt = "Must use CF long-form name \"%s\" instead of the short-form name \"%s\" when configured with %s=true";
+ String msg = String.format(fmt, shortCfNameMap.inverse().get(longName), longName, SHORT_CF_NAMES.getName());
+ throw new PermanentBackendException(msg);
+ }
+ s = longName;
+ logger.debug("Kept default CF name \"{}\" because it has no associated short form", s);
+ }
+ return s;
+ }
+
+ private TableDescriptor ensureTableExists(String tableName, String initialCFName, int ttlInSeconds) throws BackendException {
+ AdminMask adm = null;
+
+ TableDescriptor desc;
+
+ try { // Create our table, if necessary
+ adm = getAdminInterface();
+ /*
+ * Some HBase versions/impls respond badly to attempts to create a
+ * table without at least one CF. See #661. Creating a CF along with
+ * the table avoids HBase carping.
+ */
+ if (adm.tableExists(tableName)) {
+ desc = adm.getTableDescriptor(tableName);
+ // Check and warn if long and short cf names are mixedly used for the same table.
+ if (shortCfNames && initialCFName.equals(shortCfNameMap.get(SYSTEM_PROPERTIES_STORE_NAME))) {
+ String longCFName = shortCfNameMap.inverse().get(initialCFName);
+ if (desc.getColumnFamily(Bytes.toBytes(longCFName)) != null) {
+ logger.warn("Configuration {}=true, but the table \"{}\" already has column family with long name \"{}\".",
+ SHORT_CF_NAMES.getName(), tableName, longCFName);
+ logger.warn("Check {} configuration.", SHORT_CF_NAMES.getName());
+ }
+ }
+ else if (!shortCfNames && initialCFName.equals(SYSTEM_PROPERTIES_STORE_NAME)) {
+ String shortCFName = shortCfNameMap.get(initialCFName);
+ if (desc.getColumnFamily(Bytes.toBytes(shortCFName)) != null) {
+ logger.warn("Configuration {}=false, but the table \"{}\" already has column family with short name \"{}\".",
+ SHORT_CF_NAMES.getName(), tableName, shortCFName);
+ logger.warn("Check {} configuration.", SHORT_CF_NAMES.getName());
+ }
+ }
+ } else {
+ desc = createTable(tableName, initialCFName, ttlInSeconds, adm);
+ }
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ } finally {
+ IOUtils.closeQuietly(adm);
+ }
+
+ return desc;
+ }
+
+ private TableDescriptor createTable(String tableName, String cfName, int ttlInSeconds, AdminMask adm) throws IOException {
+ TableDescriptor desc = compat.newTableDescriptor(tableName);
+
+ ColumnFamilyDescriptor cdesc = ColumnFamilyDescriptorBuilder.of(cfName);
+ cdesc = setCFOptions(cdesc, ttlInSeconds);
+
+ desc = compat.addColumnFamilyToTableDescriptor(desc, cdesc);
+
+ int count; // total regions to create
+ String src;
+
+ if (MIN_REGION_COUNT <= (count = regionCount)) {
+ src = "region count configuration";
+ } else if (0 < regionsPerServer &&
+ MIN_REGION_COUNT <= (count = regionsPerServer * adm.getEstimatedRegionServerCount())) {
+ src = "ClusterStatus server count";
+ } else {
+ count = -1;
+ src = "default";
+ }
+
+ if (MIN_REGION_COUNT < count) {
+ adm.createTable(desc, getStartKey(count), getEndKey(count), count);
+ logger.debug("Created table {} with region count {} from {}", tableName, count, src);
+ } else {
+ adm.createTable(desc);
+ logger.debug("Created table {} with default start key, end key, and region count", tableName);
+ }
+
+ return desc;
+ }
+
+ /**
+ * <p/>
+ * From the {@code createTable} javadoc:
+ * "The start key specified will become the end key of the first region of
+ * the table, and the end key specified will become the start key of the
+ * last region of the table (the first region has a null start key and
+ * the last region has a null end key)"
+ * <p/>
+ * To summarize, the {@code createTable} argument called "startKey" is
+ * actually the end key of the first region.
+ */
+ private byte[] getStartKey(int regionCount) {
+ ByteBuffer regionWidth = ByteBuffer.allocate(4);
+ regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount)).flip();
+ return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
+ }
+
+ /**
+ * Companion to {@link #getStartKey(int)}. See its javadoc for details.
+ */
+ private byte[] getEndKey(int regionCount) {
+ ByteBuffer regionWidth = ByteBuffer.allocate(4);
+ regionWidth.putInt((int)(((1L << 32) - 1L) / regionCount * (regionCount - 1))).flip();
+ return StaticArrayBuffer.of(regionWidth).getBytes(0, 4);
+ }
+
+ private void ensureColumnFamilyExists(String tableName, String columnFamily, int ttlInSeconds) throws BackendException {
+ AdminMask adm = null;
+ try {
+ adm = getAdminInterface();
+ TableDescriptor desc = ensureTableExists(tableName, columnFamily, ttlInSeconds);
+
+ Preconditions.checkNotNull(desc);
+
+ ColumnFamilyDescriptor cf = desc.getColumnFamily(Bytes.toBytes(columnFamily));
+
+ // Create our column family, if necessary
+ if (cf == null) {
+ try {
+ if (!adm.isTableDisabled(tableName)) {
+ adm.disableTable(tableName);
+ }
+ } catch (TableNotEnabledException e) {
+ logger.debug("Table {} already disabled", tableName);
+ } catch (IOException e) {
+ throw new TemporaryBackendException(e);
+ }
+
+ try {
+ ColumnFamilyDescriptor cdesc = ColumnFamilyDescriptorBuilder.of(columnFamily);
+
+ setCFOptions(cdesc, ttlInSeconds);
+
+ adm.addColumn(tableName, cdesc);
+
+ try {
+ logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
+ Thread.sleep(1000L);
+ } catch (InterruptedException ie) {
+ throw new TemporaryBackendException(ie);
+ }
+
+ adm.enableTable(tableName);
+ } catch (TableNotFoundException ee) {
+ logger.error("TableNotFoundException", ee);
+ throw new PermanentBackendException(ee);
+ } catch (org.apache.hadoop.hbase.TableExistsException ee) {
+ logger.debug("Swallowing exception {}", ee);
+ } catch (IOException ee) {
+ throw new TemporaryBackendException(ee);
+ }
+ }
+ } finally {
+ IOUtils.closeQuietly(adm);
+ }
+ }
+
+ private ColumnFamilyDescriptor setCFOptions(ColumnFamilyDescriptor cdesc, int ttlInSeconds) {
+ ColumnFamilyDescriptor ret = null;
+
+ if (null != compression && !compression.equals(COMPRESSION_DEFAULT)) {
+ ret = compat.setCompression(cdesc, compression);
+ }
+
+ if (ttlInSeconds > 0) {
+ ret = ColumnFamilyDescriptorBuilder.newBuilder(cdesc).setTimeToLive(ttlInSeconds).build();
+ }
+
+ return ret;
+ }
+
+ /**
+ * Convert JanusGraph internal Mutation representation into HBase native commands.
+ *
+ * @param mutations Mutations to convert into HBase commands.
+ * @param putTimestamp The timestamp to use for Put commands.
+ * @param delTimestamp The timestamp to use for Delete commands.
+ * @return Commands sorted by key converted from JanusGraph internal representation.
+ * @throws org.janusgraph.diskstorage.PermanentBackendException
+ */
+ @VisibleForTesting
+ Map<StaticBuffer, Pair<List<Put>, Delete>> convertToCommands(Map<String, Map<StaticBuffer, KCVMutation>> mutations,
+ final long putTimestamp,
+ final long delTimestamp) throws PermanentBackendException {
+ // A map of rowkey to commands (list of Puts, Delete)
+ final Map<StaticBuffer, Pair<List<Put>, Delete>> commandsPerKey = new HashMap<>();
+
+ for (Map.Entry<String, Map<StaticBuffer, KCVMutation>> entry : mutations.entrySet()) {
+
+ String cfString = getCfNameForStoreName(entry.getKey());
+ byte[] cfName = Bytes.toBytes(cfString);
+
+ for (Map.Entry<StaticBuffer, KCVMutation> m : entry.getValue().entrySet()) {
+ final byte[] key = m.getKey().as(StaticBuffer.ARRAY_FACTORY);
+ KCVMutation mutation = m.getValue();
+
+ Pair<List<Put>, Delete> commands = commandsPerKey.get(m.getKey());
+
+ // The firt time we go through the list of input <rowkey, KCVMutation>,
+ // create the holder for a particular rowkey
+ if (commands == null) {
+ commands = new Pair<>();
+ // List of all the Puts for this rowkey, including the ones without TTL and with TTL.
+ final List<Put> putList = new ArrayList<>();
+ commands.setFirst(putList);
+ commandsPerKey.put(m.getKey(), commands);
+ }
+
+ if (mutation.hasDeletions()) {
+ if (commands.getSecond() == null) {
+ Delete d = new Delete(key);
+ compat.setTimestamp(d, delTimestamp);
+ commands.setSecond(d);
+ }
+
+ for (StaticBuffer b : mutation.getDeletions()) {
+ // commands.getSecond() is a Delete for this rowkey.
+ commands.getSecond().addColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp);
+ }
+ }
+
+ if (mutation.hasAdditions()) {
+ // All the entries (column cells) with the rowkey use this one Put, except the ones with TTL.
+ final Put putColumnsWithoutTtl = new Put(key, putTimestamp);
+ // At the end of this loop, there will be one Put entry in the commands.getFirst() list that
+ // contains all additions without TTL set, and possible multiple Put entries for columns
+ // that have TTL set.
+ for (Entry e : mutation.getAdditions()) {
+
+ // Deal with TTL within the entry (column cell) first
+ // HBase cell level TTL is actually set at the Mutation/Put level.
+ // Therefore we need to construct a new Put for each entry (column cell) with TTL.
+ // We can not combine them because column cells within the same rowkey may:
+ // 1. have no TTL
+ // 2. have TTL
+ // 3. have different TTL
+ final Integer ttl = (Integer) e.getMetaData().get(EntryMetaData.TTL);
+ if (null != ttl && ttl > 0) {
+ // Create a new Put
+ Put putColumnWithTtl = new Put(key, putTimestamp);
+ addColumnToPut(putColumnWithTtl, cfName, putTimestamp, e);
+ // Convert ttl from second (JanusGraph TTL) to millisec (HBase TTL)
+ // @see JanusGraphManagement#setTTL(JanusGraphSchemaType, Duration)
+ // Cast Put to Mutation for backward compatibility with HBase 0.98.x
+ // HBase supports cell-level TTL for versions 0.98.6 and above.
+ ((Mutation) putColumnWithTtl).setTTL(ttl * 1000);
+ // commands.getFirst() is the list of Puts for this rowkey. Add this
+ // Put column with TTL to the list.
+ commands.getFirst().add(putColumnWithTtl);
+ } else {
+ addColumnToPut(putColumnsWithoutTtl, cfName, putTimestamp, e);
+ }
+ }
+ // If there were any mutations without TTL set, add them to commands.getFirst()
+ if (!putColumnsWithoutTtl.isEmpty()) {
+ commands.getFirst().add(putColumnsWithoutTtl);
+ }
+ }
+ }
+ }
+
+ return commandsPerKey;
+ }
+
+ private void addColumnToPut(Put p, byte[] cfName, long putTimestamp, Entry e) {
+ p.addColumn(cfName, e.getColumnAs(StaticBuffer.ARRAY_FACTORY), putTimestamp,
+ e.getValueAs(StaticBuffer.ARRAY_FACTORY));
+ }
+
+ private String getCfNameForStoreName(String storeName) throws PermanentBackendException {
+ return shortCfNames ? shortenCfName(shortCfNameMap, storeName) : storeName;
+ }
+
+ private void checkConfigDeprecation(org.janusgraph.diskstorage.configuration.Configuration config) {
+ if (config.has(GraphDatabaseConfiguration.STORAGE_PORT)) {
+ logger.warn("The configuration property {} is ignored for HBase. Set hbase.zookeeper.property.clientPort in hbase-site.xml or {}.hbase.zookeeper.property.clientPort in JanusGraph's configuration file.",
+ ConfigElement.getPath(GraphDatabaseConfiguration.STORAGE_PORT), ConfigElement.getPath(HBASE_CONFIGURATION_NAMESPACE));
+ }
+ }
+
+ private AdminMask getAdminInterface() {
+ try {
+ return cnx.getAdmin();
+ } catch (IOException e) {
+ throw new JanusGraphException(e);
+ }
+ }
+
+ private String determineTableName(org.janusgraph.diskstorage.configuration.Configuration config) {
+ if ((!config.has(HBASE_TABLE)) && (config.has(GRAPH_NAME))) {
+ return config.get(GRAPH_NAME);
+ }
+ return config.get(HBASE_TABLE);
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseTransaction.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseTransaction.java
new file mode 100644
index 0000000..3b0d271
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HBaseTransaction.java
@@ -0,0 +1,31 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.janusgraph.diskstorage.BaseTransactionConfig;
+import org.janusgraph.diskstorage.common.AbstractStoreTransaction;
+
+/**
+ * This class overrides and adds nothing compared with
+ * {@link org.janusgraph.diskstorage.locking.consistentkey.ExpectedValueCheckingTransaction}; however, it creates a transaction type specific
+ * to HBase, which lets us check for user errors like passing a Cassandra
+ * transaction into a HBase method.
+ */
+public class HBaseTransaction extends AbstractStoreTransaction {
+
+ public HBaseTransaction(final BaseTransactionConfig config) {
+ super(config);
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HConnection2_0.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HConnection2_0.java
new file mode 100644
index 0000000..66b8642
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HConnection2_0.java
@@ -0,0 +1,58 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+
+import java.io.IOException;
+import java.util.List;
+
+public class HConnection2_0 implements ConnectionMask
+{
+
+ private final Connection cnx;
+
+ public HConnection2_0(Connection cnx)
+ {
+ this.cnx = cnx;
+ }
+
+ @Override
+ public TableMask getTable(String name) throws IOException
+ {
+ return new HTable2_0(cnx.getTable(TableName.valueOf(name)));
+ }
+
+ @Override
+ public AdminMask getAdmin() throws IOException
+ {
+ return new HBaseAdmin2_0(cnx.getAdmin());
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ cnx.close();
+ }
+
+ @Override
+ public List<HRegionLocation> getRegionLocations(String tableName)
+ throws IOException
+ {
+ return this.cnx.getRegionLocator(TableName.valueOf(tableName)).getAllRegionLocations();
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HTable2_0.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HTable2_0.java
new file mode 100644
index 0000000..0b4643a
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/HTable2_0.java
@@ -0,0 +1,60 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+
+import java.io.IOException;
+import java.util.List;
+
+public class HTable2_0 implements TableMask
+{
+ private final Table table;
+
+ public HTable2_0(Table table)
+ {
+ this.table = table;
+ }
+
+ @Override
+ public ResultScanner getScanner(Scan filter) throws IOException
+ {
+ return table.getScanner(filter);
+ }
+
+ @Override
+ public Result[] get(List<Get> gets) throws IOException
+ {
+ return table.get(gets);
+ }
+
+ @Override
+ public void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException
+ {
+ table.batch(writes, results);
+ /* table.flushCommits(); not needed anymore */
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ table.close();
+ }
+}
diff --git a/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/TableMask.java b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/TableMask.java
new file mode 100644
index 0000000..0309c39
--- /dev/null
+++ b/graphdb/janus-hbase2/src/main/java/org/janusgraph/diskstorage/hbase2/TableMask.java
@@ -0,0 +1,45 @@
+// Copyright 2017 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Copyright DataStax, Inc.
+ * <p>
+ * Please see the included license file for details.
+ */
+package org.janusgraph.diskstorage.hbase2;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This interface hides ABI/API breaking changes that HBase has made to its Table/HTableInterface over the course
+ * of development from 0.94 to 1.0 and beyond.
+ */
+public interface TableMask extends Closeable
+{
+
+ ResultScanner getScanner(Scan filter) throws IOException;
+
+ Result[] get(List<Get> gets) throws IOException;
+
+ void batch(List<Row> writes, Object[] results) throws IOException, InterruptedException;
+
+}
diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml
index 5d491e8..543e340 100644
--- a/graphdb/janus/pom.xml
+++ b/graphdb/janus/pom.xml
@@ -55,6 +55,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-janusgraph-hbase2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
<artifactId>atlas-testtools</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 80e9cc3..2f367d5 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -36,6 +36,7 @@ import org.janusgraph.core.JanusGraphException;
import org.janusgraph.core.JanusGraphFactory;
import org.janusgraph.core.schema.JanusGraphManagement;
import org.janusgraph.diskstorage.StandardIndexProvider;
+import org.janusgraph.diskstorage.StandardStoreManager;
import org.janusgraph.diskstorage.solr.Solr6Index;
import org.janusgraph.graphdb.database.serialize.attribute.SerializableSerializer;
import org.janusgraph.graphdb.tinkerpop.JanusGraphIoRegistry;
@@ -104,9 +105,31 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
}
static {
+ addHBase2Support();
+
addSolr6Index();
}
+ private static void addHBase2Support() {
+ try {
+ Field field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES");
+ field.setAccessible(true);
+
+ Field modifiersField = Field.class.getDeclaredField("modifiers");
+ modifiersField.setAccessible(true);
+ modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
+
+ Map<String, String> customMap = new HashMap<>(StandardStoreManager.getAllManagerClasses());
+ customMap.put("hbase2", org.janusgraph.diskstorage.hbase2.HBaseStoreManager.class.getName());
+ ImmutableMap<String, String> immap = ImmutableMap.copyOf(customMap);
+ field.set(null, immap);
+
+ LOG.debug("Injected HBase2 support - {}", org.janusgraph.diskstorage.hbase2.HBaseStoreManager.class.getName());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private static void addSolr6Index() {
try {
Field field = StandardIndexProvider.class.getDeclaredField("ALL_MANAGER_CLASSES");
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasElementPropertyConfig.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasElementPropertyConfig.java
index 2945cf1..abf65ac 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasElementPropertyConfig.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasElementPropertyConfig.java
@@ -24,8 +24,6 @@ import java.util.Set;
/**
* Configure how the GraphSON utility treats edge and vertex properties.
- *
- * @author Stephen Mallette (http://stephen.genoprime.com)
*/
public class AtlasElementPropertyConfig {
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONMode.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONMode.java
index 4857378..f9fdc64 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONMode.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONMode.java
@@ -19,8 +19,6 @@ package org.apache.atlas.repository.graphdb.janus.graphson;
/**
* Modes of operation of the GraphSONUtility.
- *
- * @author Stephen Mallette
*/
public enum AtlasGraphSONMode {
/**
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONTokens.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONTokens.java
index d031708..b360804 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONTokens.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONTokens.java
@@ -17,9 +17,6 @@
*/
package org.apache.atlas.repository.graphdb.janus.graphson;
-/**
- * @author Stephen Mallette (http://stephen.genoprime.com)
- */
public final class AtlasGraphSONTokens {
private AtlasGraphSONTokens() {}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONUtility.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONUtility.java
index b3c9095..2bd45c6 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONUtility.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/graphson/AtlasGraphSONUtility.java
@@ -48,8 +48,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
*
* Helps write individual graph elements to TinkerPop JSON format known as
* GraphSON.
- *
- * @author Stephen Mallette (http://stephen.genoprime.com)
*/
public final class AtlasGraphSONUtility {
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index d7097d2..b300668 100644
--- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -17,8 +17,35 @@
*/
package org.janusgraph.diskstorage.solr;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
+import static org.janusgraph.diskstorage.solr.SolrIndex.*;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.UncheckedIOException;
+import java.lang.reflect.Constructor;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.TimeZone;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
@@ -96,49 +123,8 @@ import org.janusgraph.graphdb.types.ParameterType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.UncheckedIOException;
-import java.lang.reflect.Constructor;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.time.Instant;
-import java.util.AbstractMap.SimpleEntry;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Spliterator;
-import java.util.Spliterators;
-import java.util.TimeZone;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-
-import static org.janusgraph.diskstorage.solr.SolrIndex.DYNAMIC_FIELDS;
-import static org.janusgraph.diskstorage.solr.SolrIndex.HTTP_ALLOW_COMPRESSION;
-import static org.janusgraph.diskstorage.solr.SolrIndex.HTTP_CONNECTION_TIMEOUT;
-import static org.janusgraph.diskstorage.solr.SolrIndex.HTTP_GLOBAL_MAX_CONNECTIONS;
-import static org.janusgraph.diskstorage.solr.SolrIndex.HTTP_MAX_CONNECTIONS_PER_HOST;
-import static org.janusgraph.diskstorage.solr.SolrIndex.HTTP_URLS;
-import static org.janusgraph.diskstorage.solr.SolrIndex.KERBEROS_ENABLED;
-import static org.janusgraph.diskstorage.solr.SolrIndex.KEY_FIELD_NAMES;
-import static org.janusgraph.diskstorage.solr.SolrIndex.MAX_SHARDS_PER_NODE;
-import static org.janusgraph.diskstorage.solr.SolrIndex.NUM_SHARDS;
-import static org.janusgraph.diskstorage.solr.SolrIndex.REPLICATION_FACTOR;
-import static org.janusgraph.diskstorage.solr.SolrIndex.SOLR_DEFAULT_CONFIG;
-import static org.janusgraph.diskstorage.solr.SolrIndex.SOLR_MODE;
-import static org.janusgraph.diskstorage.solr.SolrIndex.SOLR_NS;
-import static org.janusgraph.diskstorage.solr.SolrIndex.TTL_FIELD;
-import static org.janusgraph.diskstorage.solr.SolrIndex.WAIT_SEARCHER;
-import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INDEX_MAX_RESULT_SET_SIZE;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
/**
* NOTE: Copied from JanusGraph for supporting Kerberos and adding support for multiple zookeeper clients. Do not change
@@ -193,6 +179,9 @@ public class Solr6Index implements IndexProvider {
private final boolean kerberosEnabled;
public Solr6Index(final Configuration config) throws BackendException {
+ // Add Kerberos-enabled SolrHttpClientBuilder
+ HttpClientUtil.setHttpClientBuilder(new Krb5HttpClientBuilder().getBuilder());
+
Preconditions.checkArgument(config!=null);
configuration = config;
mode = Mode.parse(config.get(SOLR_MODE));
diff --git a/graphdb/pom.xml b/graphdb/pom.xml
index 707b13c..499b411 100644
--- a/graphdb/pom.xml
+++ b/graphdb/pom.xml
@@ -36,6 +36,7 @@
<module>api</module>
<module>common</module>
<module>graphdb-impls</module>
+ <module>janus-hbase2</module>
<module>janus</module>
</modules>
diff --git a/intg/pom.xml b/intg/pom.xml
index 31361e2..7582549 100644
--- a/intg/pom.xml
+++ b/intg/pom.xml
@@ -43,6 +43,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -89,6 +93,12 @@
</dependency>
<dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>${commons-conf.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
index 1d24ee4..01af49c 100644
--- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -44,6 +44,19 @@ public final class ApplicationProperties extends PropertiesConfiguration {
public static final String APPLICATION_PROPERTIES = "atlas-application.properties";
+ public static final String GRAPHDB_BACKEND_CONF = "atlas.graphdb.backend";
+ public static final String STORAGE_BACKEND_CONF = "atlas.graph.storage.backend";
+ public static final String INDEX_BACKEND_CONF = "atlas.graph.index.search.backend";
+ public static final String INDEX_MAP_NAME_CONF = "atlas.graph.index.search.map-name";
+ public static final String SOLR_WAIT_SEARCHER_CONF = "atlas.graph.index.search.solr.wait-searcher";
+ public static final String GRAPHBD_BACKEND_JANUS = "janus";
+ public static final String STORAGE_BACKEND_HBASE = "hbase";
+ public static final String STORAGE_BACKEND_HBASE2 = "hbase2";
+ public static final String INDEX_BACKEND_SOLR = "solr";
+ public static final String DEFAULT_GRAPHDB_BACKEND = GRAPHBD_BACKEND_JANUS;
+ public static final boolean DEFAULT_SOLR_WAIT_SEARCHER = true;
+ public static final boolean DEFAULT_INDEX_MAP_NAME = false;
+
public static final SimpleEntry<String, String> DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true");
public static final SimpleEntry<String, String> DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20");
public static final SimpleEntry<String, String> DB_CACHE_SIZE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-size", "0.5");
@@ -248,6 +261,64 @@ public final class ApplicationProperties extends PropertiesConfiguration {
}
private void setDefaults() {
+ String graphDbBackend = getString(GRAPHDB_BACKEND_CONF);
+
+ if (StringUtils.isEmpty(graphDbBackend)) {
+ graphDbBackend = DEFAULT_GRAPHDB_BACKEND;
+
+ clearPropertyDirect(GRAPHDB_BACKEND_CONF);
+ addPropertyDirect(GRAPHDB_BACKEND_CONF, graphDbBackend);
+ LOG.info("No graphdb backend specified. Will use '" + graphDbBackend + "'");
+
+ // The below default values for storage backend, index backend and solr-wait-searcher
+ // should be removed once ambari change to handle them is committed.
+ clearPropertyDirect(STORAGE_BACKEND_CONF);
+ addPropertyDirect(STORAGE_BACKEND_CONF, STORAGE_BACKEND_HBASE2);
+ LOG.info("Using storage backend '" + STORAGE_BACKEND_HBASE2 + "'");
+
+ clearPropertyDirect(INDEX_BACKEND_CONF);
+ addPropertyDirect(INDEX_BACKEND_CONF, INDEX_BACKEND_SOLR);
+ LOG.info("Using index backend '" + INDEX_BACKEND_SOLR + "'");
+
+ clearPropertyDirect(SOLR_WAIT_SEARCHER_CONF);
+ addPropertyDirect(SOLR_WAIT_SEARCHER_CONF, DEFAULT_SOLR_WAIT_SEARCHER);
+ LOG.info("Setting solr-wait-searcher property '" + DEFAULT_SOLR_WAIT_SEARCHER + "'");
+
+ clearPropertyDirect(INDEX_MAP_NAME_CONF);
+ addPropertyDirect(INDEX_MAP_NAME_CONF, DEFAULT_INDEX_MAP_NAME);
+ LOG.info("Setting index.search.map-name property '" + DEFAULT_INDEX_MAP_NAME + "'");
+ }
+
+ String storageBackend = getString(STORAGE_BACKEND_CONF);
+
+ if (StringUtils.isEmpty(storageBackend)) {
+ if (graphDbBackend.contains(GRAPHBD_BACKEND_JANUS)) {
+ storageBackend = STORAGE_BACKEND_HBASE2;
+ }
+
+ if (StringUtils.isNotEmpty(storageBackend)) {
+ clearPropertyDirect(STORAGE_BACKEND_CONF);
+ addPropertyDirect(STORAGE_BACKEND_CONF, storageBackend);
+
+ LOG.info("No storage backend specified. Will use '" + storageBackend + "'");
+ }
+ }
+
+ String indexBackend = getString(INDEX_BACKEND_CONF);
+
+ if (StringUtils.isEmpty(indexBackend)) {
+ if (graphDbBackend.contains(GRAPHBD_BACKEND_JANUS)) {
+ indexBackend = INDEX_BACKEND_SOLR;
+ }
+
+ if (StringUtils.isNotEmpty(indexBackend)) {
+ clearPropertyDirect(INDEX_BACKEND_CONF);
+ addPropertyDirect(INDEX_BACKEND_CONF, indexBackend);
+
+ LOG.info("No index backend specified. Will use '" + indexBackend + "'");
+ }
+ }
+
setDbCacheConfDefaults();
}
diff --git a/pom.xml b/pom.xml
index d9b2c9d..5334478 100644
--- a/pom.xml
+++ b/pom.xml
@@ -557,7 +557,7 @@
<activeByDefault>false</activeByDefault>
</activation>
<properties>
- <graph.storage.backend>hbase</graph.storage.backend>
+ <graph.storage.backend>hbase2</graph.storage.backend>
<graph.index.backend>solr</graph.index.backend>
<solr.zk.address>localhost:9983</solr.zk.address>
<graph.storage.hostname>localhost</graph.storage.hostname>
@@ -616,6 +616,7 @@
</property>
</activation>
<properties>
+ <!-- Define graph dependency type/version -->
<graphGroup>org.apache.atlas</graphGroup>
<graphArtifact>atlas-graphdb-janus</graphArtifact>
<skipDocs>false</skipDocs>
@@ -649,15 +650,20 @@
<jersey.version>1.19</jersey.version>
<jsr.version>1.1</jsr.version>
- <hadoop.version>2.7.1</hadoop.version>
- <hbase.version>1.1.2</hbase.version>
- <solr.version>5.5.1</solr.version>
- <kafka.version>1.0.0</kafka.version>
- <elasticsearch.version>5.6.4</elasticsearch.version>
+ <janus.version>0.3.1</janus.version>
+ <hadoop.version>3.1.1</hadoop.version>
+ <hbase.version>2.0.2</hbase.version>
+ <solr.version>7.5.0</solr.version>
+ <hive.version>3.1.0</hive.version>
+ <kafka.version>2.0.0</kafka.version>
<kafka.scala.binary.version>2.11</kafka.scala.binary.version>
- <curator.version>2.11.0</curator.version>
+ <calcite.version>1.16.0</calcite.version>
<zookeeper.version>3.4.6</zookeeper.version>
- <janus.version>0.3.1</janus.version>
+ <falcon.version>0.8</falcon.version>
+ <sqoop.version>1.4.6.2.3.99.0-195</sqoop.version>
+ <storm.version>1.2.0</storm.version>
+ <curator.version>4.0.1</curator.version>
+ <elasticsearch.version>5.6.4</elasticsearch.version>
<json.version>3.2.11</json.version>
<log4j.version>1.2.17</log4j.version>
@@ -666,17 +672,16 @@
<gson.version>2.5</gson.version>
<fastutil.version>6.5.16</fastutil.version>
<guice.version>4.1.0</guice.version>
- <spring.version>4.3.17.RELEASE</spring.version>
- <spring.security.version>4.2.6.RELEASE</spring.security.version>
+ <spring.version>4.3.18.RELEASE</spring.version>
+ <spring.security.version>4.2.7.RELEASE</spring.security.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
- <guava.version>19.0</guava.version>
- <scala.version>2.11.12</scala.version>
+ <guava.version>25.1-jre</guava.version>
<antlr4.version>4.7</antlr4.version>
<!-- Needed for hooks -->
<aopalliance.version>1.0</aopalliance.version>
- <jackson.version>2.9.6</jackson.version>
+ <jackson.version>2.9.8</jackson.version>
<!-- Apache commons -->
<commons-conf.version>1.10</commons-conf.version>
@@ -700,11 +705,12 @@
<doxia.version>1.8</doxia.version>
<dropwizard-metrics>3.2.2</dropwizard-metrics>
<!-- hadoop.hdfs-client.version should same as hadoop version -->
- <hadoop.hdfs-client.version>2.8.1</hadoop.hdfs-client.version>
+ <hadoop.hdfs-client.version>${hadoop.version}</hadoop.hdfs-client.version>
<!-- Storm dependencies -->
<codehaus.woodstox.stax2-api.version>3.1.4</codehaus.woodstox.stax2-api.version>
<woodstox-core.version>5.0.3</woodstox-core.version>
+ <hppc.version>0.8.1</hppc.version>
<!-- Storm dependencies -->
<PermGen>64m</PermGen>
@@ -751,8 +757,6 @@
<module>notification</module>
<module>client</module>
<module>graphdb</module>
- <module>shaded/hbase-client-shaded</module>
- <module>shaded/hbase-server-shaded</module>
<module>repository</module>
<module>authorization</module>
<module>dashboardv2</module>
@@ -771,6 +775,7 @@
<module>addons/storm-bridge</module>
<module>addons/hbase-bridge-shim</module>
<module>addons/hbase-bridge</module>
+ <module>addons/hbase-testing-util</module>
<module>addons/kafka-bridge</module>
<module>distro</module>
@@ -1423,31 +1428,6 @@
<dependency>
<groupId>org.apache.atlas</groupId>
- <artifactId>atlas-hbase-client-shaded</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.atlas</groupId>
- <artifactId>atlas-hbase-server-shaded</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.atlas</groupId>
<artifactId>atlas-buildtools</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/repository/pom.xml b/repository/pom.xml
index 8198610..42e1679 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -138,18 +138,25 @@
</dependency>
<dependency>
- <groupId>org.apache.atlas</groupId>
- <artifactId>atlas-hbase-client-shaded</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.atlas</groupId>
- <artifactId>atlas-hbase-server-shaded</artifactId>
- <scope>test</scope>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -183,7 +190,7 @@
<dependency>
<groupId>com.datastax.cassandra</groupId>
<artifactId>cassandra-driver-core</artifactId>
- <version>3.1.4</version>
+ <version>3.2.0</version>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
@@ -211,6 +218,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.carrotsearch</groupId>
+ <artifactId>hppc</artifactId>
+ <version>${hppc.version}</version>
+ </dependency>
</dependencies>
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
index 5f01293..6e8dbe9 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
@@ -22,21 +22,13 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
-import org.apache.atlas.EntityAuditEvent.EntityAuditAction;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
-import org.apache.atlas.model.audit.EntityAuditEventV2;
-import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
-import org.apache.atlas.model.instance.AtlasClassification;
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
-import org.apache.atlas.repository.converters.AtlasInstanceConverter;
-import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.v1.model.instance.Referenceable;
+import org.apache.atlas.model.audit.EntityAuditEventV2;
+import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -62,25 +54,18 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
-import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_ADD;
-import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_DELETE;
-import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_UPDATE;
-import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TERM_ADD;
-import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TERM_DELETE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType.ENTITY_AUDIT_V1;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType.ENTITY_AUDIT_V2;
-import static org.apache.atlas.repository.audit.EntityAuditListener.getV2AuditPrefix;
/**
* HBase based repository for entity audit events
@@ -102,22 +87,45 @@ import static org.apache.atlas.repository.audit.EntityAuditListener.getV2AuditPr
public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditRepository {
private static final Logger LOG = LoggerFactory.getLogger(HBaseBasedAuditRepository.class);
- public static final String CONFIG_TABLE_NAME = CONFIG_PREFIX + ".hbase.tablename";
+ public static final String CONFIG_PREFIX = "atlas.audit";
+ public static final String CONFIG_TABLE_NAME = CONFIG_PREFIX + ".hbase.tablename";
public static final String DEFAULT_TABLE_NAME = "ATLAS_ENTITY_AUDIT_EVENTS";
- public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt");
- public static final byte[] COLUMN_ACTION = Bytes.toBytes("a");
- public static final byte[] COLUMN_DETAIL = Bytes.toBytes("d");
- public static final byte[] COLUMN_USER = Bytes.toBytes("u");
- public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("f");
- public static final byte[] COLUMN_TYPE = Bytes.toBytes("t");
+ public static final String CONFIG_PERSIST_ENTITY_DEFINITION = CONFIG_PREFIX + ".persistEntityDefinition";
+
+ public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt");
+ public static final byte[] COLUMN_ACTION = Bytes.toBytes("a");
+ public static final byte[] COLUMN_DETAIL = Bytes.toBytes("d");
+ public static final byte[] COLUMN_USER = Bytes.toBytes("u");
+ public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("f");
+
+ private static final String AUDIT_REPOSITORY_MAX_SIZE_PROPERTY = "atlas.hbase.client.keyvalue.maxsize";
+ private static final String AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY = "atlas.audit.hbase.entity";
+ private static final String FIELD_SEPARATOR = ":";
+ private static final long ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE = 1024 * 1024;
+ private static Configuration APPLICATION_PROPERTIES = null;
+
+ private static boolean persistEntityDefinition;
+ private Map<String, List<String>> auditExcludedAttributesCache = new HashMap<>();
+
+ static {
+ try {
+ persistEntityDefinition = ApplicationProperties.get().getBoolean(CONFIG_PERSIST_ENTITY_DEFINITION, false);
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
+ }
private TableName tableName;
private Connection connection;
- private final AtlasInstanceConverter instanceConverter;
- @Inject
- public HBaseBasedAuditRepository(AtlasInstanceConverter instanceConverter) {
- this.instanceConverter = instanceConverter;
+ /**
+ * Add events to the event repository
+ * @param events events to be added
+ * @throws AtlasException
+ */
+ @Override
+ public void putEventsV1(EntityAuditEvent... events) throws AtlasException {
+ putEventsV1(Arrays.asList(events));
}
/**
@@ -149,8 +157,6 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
addColumn(put, COLUMN_ACTION, event.getAction());
addColumn(put, COLUMN_USER, event.getUser());
addColumn(put, COLUMN_DETAIL, event.getDetails());
- addColumn(put, COLUMN_TYPE, ENTITY_AUDIT_V1);
-
if (persistEntityDefinition) {
addColumn(put, COLUMN_DEFINITION, event.getEntityDefinitionString());
}
@@ -167,6 +173,11 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
}
@Override
+ public void putEventsV2(EntityAuditEventV2... events) throws AtlasBaseException {
+ putEventsV2(Arrays.asList(events));
+ }
+
+ @Override
public void putEventsV2(List<EntityAuditEventV2> events) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("Putting {} events", events.size());
@@ -190,7 +201,6 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
addColumn(put, COLUMN_ACTION, event.getAction());
addColumn(put, COLUMN_USER, event.getUser());
addColumn(put, COLUMN_DETAIL, event.getDetails());
- addColumn(put, COLUMN_TYPE, ENTITY_AUDIT_V2);
if (persistEntityDefinition) {
addColumn(put, COLUMN_DEFINITION, event.getEntityDefinitionString());
@@ -260,11 +270,14 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
event.setUser(getResultString(result, COLUMN_USER));
event.setAction(EntityAuditActionV2.fromString(getResultString(result, COLUMN_ACTION)));
- event.setDetails(getEntityDetails(result));
- event.setType(getAuditType(result));
+ event.setDetails(getResultString(result, COLUMN_DETAIL));
if (persistEntityDefinition) {
- event.setEntityDefinition(getEntityDefinition(result));
+ String colDef = getResultString(result, COLUMN_DEFINITION);
+
+ if (colDef != null) {
+ event.setEntityDefinition(colDef);
+ }
}
events.add(event);
@@ -287,92 +300,16 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
}
}
- private String getEntityDefinition(Result result) throws AtlasBaseException {
- String ret = getResultString(result, COLUMN_DEFINITION);
-
- if (getAuditType(result) != ENTITY_AUDIT_V2) {
- Referenceable referenceable = AtlasType.fromV1Json(ret, Referenceable.class);
- AtlasEntity entity = toAtlasEntity(referenceable);
-
- ret = AtlasType.toJson(entity);
- }
-
- return ret;
- }
-
- private String getEntityDetails(Result result) throws AtlasBaseException {
- String ret;
-
- if (getAuditType(result) == ENTITY_AUDIT_V2) {
- ret = getResultString(result, COLUMN_DETAIL);
- } else {
- // convert v1 audit detail to v2
- ret = getV2Details(result);
- }
-
- return ret;
- }
-
- private EntityAuditType getAuditType(Result result) {
- String typeString = getResultString(result, COLUMN_TYPE);
- EntityAuditType ret = (typeString != null) ? EntityAuditType.valueOf(typeString) : ENTITY_AUDIT_V1;
... 1318 lines suppressed ...