You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2017/09/15 21:20:19 UTC
[02/25] hbase git commit: HBASE-18725 [C++] Install header files as
well as library
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/SecureBulkLoad.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/SecureBulkLoad.proto b/hbase-native-client/src/hbase/if/SecureBulkLoad.proto
new file mode 100644
index 0000000..814735b
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/SecureBulkLoad.proto
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SecureBulkLoadProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import 'HBase.proto';
+import 'Client.proto';
+
+message SecureBulkLoadHFilesRequest {
+ repeated BulkLoadHFileRequest.FamilyPath family_path = 1;
+ optional bool assign_seq_num = 2;
+ required DelegationToken fs_token = 3;
+ required string bulk_token = 4;
+}
+
+message SecureBulkLoadHFilesResponse {
+ required bool loaded = 1;
+}
+
+message DelegationToken {
+ optional bytes identifier = 1;
+ optional bytes password = 2;
+ optional string kind = 3;
+ optional string service = 4;
+}
+
+message PrepareBulkLoadRequest {
+ required TableName table_name = 1;
+}
+
+message PrepareBulkLoadResponse {
+ required string bulk_token = 1;
+}
+
+message CleanupBulkLoadRequest {
+ required string bulk_token = 1;
+
+}
+
+message CleanupBulkLoadResponse {
+}
+
+service SecureBulkLoadService {
+ rpc PrepareBulkLoad(PrepareBulkLoadRequest)
+ returns (PrepareBulkLoadResponse);
+
+ rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest)
+ returns (SecureBulkLoadHFilesResponse);
+
+ rpc CleanupBulkLoad(CleanupBulkLoadRequest)
+ returns (CleanupBulkLoadResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/Snapshot.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/Snapshot.proto b/hbase-native-client/src/hbase/if/Snapshot.proto
new file mode 100644
index 0000000..ae1a1e6
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/Snapshot.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SnapshotProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "FS.proto";
+import "HBase.proto";
+
+message SnapshotFileInfo {
+ enum Type {
+ HFILE = 1;
+ WAL = 2;
+ }
+
+ required Type type = 1;
+
+ optional string hfile = 3;
+
+ optional string wal_server = 4;
+ optional string wal_name = 5;
+}
+
+message SnapshotRegionManifest {
+ optional int32 version = 1;
+
+ required RegionInfo region_info = 2;
+ repeated FamilyFiles family_files = 3;
+
+ message StoreFile {
+ required string name = 1;
+ optional Reference reference = 2;
+
+ // TODO: Add checksums or other fields to verify the file
+ optional uint64 file_size = 3;
+ }
+
+ message FamilyFiles {
+ required bytes family_name = 1;
+ repeated StoreFile store_files = 2;
+ }
+}
+
+message SnapshotDataManifest {
+ required TableSchema table_schema = 1;
+ repeated SnapshotRegionManifest region_manifests = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/Tracing.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/Tracing.proto b/hbase-native-client/src/hbase/if/Tracing.proto
new file mode 100644
index 0000000..5a64cfc
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/Tracing.proto
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "TracingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+//Used to pass through the information necessary to continue
+//a trace after an RPC is made. All we need is the traceid
+//(so we know the overarching trace this message is a part of), and
+//the id of the current span when this message was sent, so we know
+//what span caused the new span we will create when this message is received.
+message RPCTInfo {
+ optional int64 trace_id = 1;
+ optional int64 parent_id = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/VisibilityLabels.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/VisibilityLabels.proto b/hbase-native-client/src/hbase/if/VisibilityLabels.proto
new file mode 100644
index 0000000..d2dc44d
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/VisibilityLabels.proto
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "VisibilityLabelsProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Client.proto";
+
+message VisibilityLabelsRequest {
+ repeated VisibilityLabel visLabel = 1;
+}
+
+message VisibilityLabel {
+ required bytes label = 1;
+ optional uint32 ordinal = 2;
+}
+
+message VisibilityLabelsResponse {
+ repeated RegionActionResult result = 1;
+}
+
+message SetAuthsRequest {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message UserAuthorizations {
+ required bytes user = 1;
+ repeated uint32 auth = 2;
+}
+
+message MultiUserAuthorizations {
+ repeated UserAuthorizations userAuths = 1;
+}
+
+message GetAuthsRequest {
+ required bytes user = 1;
+}
+
+message GetAuthsResponse {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message ListLabelsRequest {
+ optional string regex = 1;
+}
+
+message ListLabelsResponse {
+ repeated bytes label = 1;
+}
+
+service VisibilityLabelsService {
+ rpc addLabels(VisibilityLabelsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc setAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc clearAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc getAuths(GetAuthsRequest)
+ returns (GetAuthsResponse);
+ rpc listLabels(ListLabelsRequest)
+ returns (ListLabelsResponse);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/WAL.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/WAL.proto b/hbase-native-client/src/hbase/if/WAL.proto
new file mode 100644
index 0000000..2061b22
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/WAL.proto
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "WALProtos";
+option java_generic_services = false;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+
+message WALHeader {
+ optional bool has_compression = 1;
+ optional bytes encryption_key = 2;
+ optional bool has_tag_compression = 3;
+ optional string writer_cls_name = 4;
+ optional string cell_codec_cls_name = 5;
+}
+
+/*
+ * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header
+ * for some KVs
+ */
+message WALKey {
+ required bytes encoded_region_name = 1;
+ required bytes table_name = 2;
+ required uint64 log_sequence_number = 3;
+ required uint64 write_time = 4;
+ /*
+ This parameter is deprecated in favor of clusters which
+ contains the list of clusters that have consumed the change.
+ It is retained so that the log created by earlier releases (0.94)
+ can be read by the newer releases.
+ */
+ optional UUID cluster_id = 5 [deprecated=true];
+
+ repeated FamilyScope scopes = 6;
+ optional uint32 following_kv_count = 7;
+
+ /*
+ This field contains the list of clusters that have
+ consumed the change
+ */
+ repeated UUID cluster_ids = 8;
+
+ optional uint64 nonceGroup = 9;
+ optional uint64 nonce = 10;
+ optional uint64 orig_sequence_number = 11;
+
+/*
+ optional CustomEntryType custom_entry_type = 9;
+
+ enum CustomEntryType {
+ COMPACTION = 0;
+ }
+*/
+}
+
+enum ScopeType {
+ REPLICATION_SCOPE_LOCAL = 0;
+ REPLICATION_SCOPE_GLOBAL = 1;
+}
+
+message FamilyScope {
+ required bytes family = 1;
+ required ScopeType scope_type = 2;
+}
+
+/**
+ * Custom WAL entries
+ */
+
+/**
+ * Special WAL entry to hold all related to a compaction.
+ * Written to WAL before completing compaction. There is
+ * sufficient info in the below message to complete later
+ * the * compaction should we fail the WAL write.
+ */
+message CompactionDescriptor {
+ required bytes table_name = 1; // TODO: WALKey already stores these, might remove
+ required bytes encoded_region_name = 2;
+ required bytes family_name = 3;
+ repeated string compaction_input = 4; // relative to store dir
+ repeated string compaction_output = 5;
+ required string store_home_dir = 6; // relative to region dir
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * Special WAL entry to hold all related to a flush.
+ */
+message FlushDescriptor {
+ enum FlushAction {
+ START_FLUSH = 0;
+ COMMIT_FLUSH = 1;
+ ABORT_FLUSH = 2;
+ CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete
+ }
+
+ message StoreFlushDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH)
+ }
+
+ required FlushAction action = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 flush_sequence_number = 4;
+ repeated StoreFlushDescriptor store_flushes = 5;
+ optional bytes region_name = 6; // full region name
+}
+
+message StoreDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string store_file = 3; // relative to store dir
+ optional uint64 store_file_size_bytes = 4; // size of store file
+}
+
+/**
+ * Special WAL entry used for writing bulk load events to WAL
+ */
+message BulkLoadDescriptor {
+ required TableName table_name = 1;
+ required bytes encoded_region_name = 2;
+ repeated StoreDescriptor stores = 3;
+ required int64 bulkload_seq_num = 4;
+}
+
+/**
+ * Special WAL entry to hold all related to a region event (open/close).
+ */
+message RegionEventDescriptor {
+ enum EventType {
+ REGION_OPEN = 0;
+ REGION_CLOSE = 1;
+ }
+
+ required EventType event_type = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 log_sequence_number = 4;
+ repeated StoreDescriptor stores = 5;
+ optional ServerName server = 6; // Server who opened the region
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * A trailer that is appended to the end of a properly closed WAL file.
+ * If missing, this is either a legacy or a corrupted WAL file.
+ * N.B. This trailer currently doesn't contain any information and we
+ * purposefully don't expose it in the WAL APIs. It's for future growth.
+ */
+message WALTrailer {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/ZooKeeper.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/ZooKeeper.proto b/hbase-native-client/src/hbase/if/ZooKeeper.proto
new file mode 100644
index 0000000..41c0e0e
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/ZooKeeper.proto
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ZNode data in hbase are serialized protobufs with a four byte
+// 'magic' 'PBUF' prefix.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ZooKeeperProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterStatus.proto";
+
+/**
+ * Content of the meta-region-server znode.
+ */
+message MetaRegionServer {
+ // The ServerName hosting the meta region currently, or destination server,
+ // if meta region is in transition.
+ required ServerName server = 1;
+ // The major version of the rpc the server speaks. This is used so that
+ // clients connecting to the cluster can have prior knowledge of what version
+ // to send to a RegionServer. AsyncHBase will use this to detect versions.
+ optional uint32 rpc_version = 2;
+
+ // State of the region transition. OPEN means fully operational 'hbase:meta'
+ optional RegionState.State state = 3;
+}
+
+/**
+ * Content of the master znode.
+ */
+message Master {
+ // The ServerName of the current Master
+ required ServerName master = 1;
+ // Major RPC version so that clients can know what version the master can accept.
+ optional uint32 rpc_version = 2;
+ optional uint32 info_port = 3;
+}
+
+/**
+ * Content of the '/hbase/running', cluster state, znode.
+ */
+message ClusterUp {
+ // If this znode is present, cluster is up. Currently
+ // the data is cluster start_date.
+ required string start_date = 1;
+}
+
+/**
+ * WAL SplitLog directory znodes have this for content. Used doing distributed
+ * WAL splitting. Holds current state and name of server that originated split.
+ */
+message SplitLogTask {
+ enum State {
+ UNASSIGNED = 0;
+ OWNED = 1;
+ RESIGNED = 2;
+ DONE = 3;
+ ERR = 4;
+ }
+ enum RecoveryMode {
+ UNKNOWN = 0;
+ LOG_SPLITTING = 1;
+ LOG_REPLAY = 2;
+ }
+ required State state = 1;
+ required ServerName server_name = 2;
+ optional RecoveryMode mode = 3 [default = UNKNOWN];
+}
+
+/**
+ * The znode that holds state of table.
+ * Deprected, table state is stored in table descriptor on HDFS.
+ */
+message DeprecatedTableState {
+ // Table's current state
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ DISABLING = 2;
+ ENABLING = 3;
+ }
+ // This is the table's state. If no znode for a table,
+ // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class
+ // for more.
+ required State state = 1 [default = ENABLED];
+}
+
+message TableCF {
+ optional TableName table_name = 1;
+ repeated bytes families = 2;
+}
+
+/**
+ * Used by replication. Holds a replication peer key.
+ */
+message ReplicationPeer {
+ // clusterkey is the concatenation of the slave cluster's
+ // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
+ required string clusterkey = 1;
+ optional string replicationEndpointImpl = 2;
+ repeated BytesBytesPair data = 3;
+ repeated NameStringPair configuration = 4;
+ repeated TableCF table_cfs = 5;
+}
+
+/**
+ * Used by replication. Holds whether enabled or disabled
+ */
+message ReplicationState {
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ }
+ required State state = 1;
+}
+
+/**
+ * Used by replication. Holds the current position in an WAL file.
+ */
+message ReplicationHLogPosition {
+ required int64 position = 1;
+}
+
+/**
+ * Used by replication. Used to lock a region server during failover.
+ */
+message ReplicationLock {
+ required string lock_owner = 1;
+}
+
+/**
+ * Metadata associated with a table lock in zookeeper
+ */
+message TableLock {
+ optional TableName table_name = 1;
+ optional ServerName lock_owner = 2;
+ optional int64 thread_id = 3;
+ optional bool is_shared = 4;
+ optional string purpose = 5;
+ optional int64 create_time = 6;
+}
+
+/**
+ * State of the switch.
+ */
+message SwitchState {
+ optional bool enabled = 1;
+}
+
+/**
+ * State for split and merge, used in hbck
+ */
+message SplitAndMergeState {
+ optional bool split_enabled = 1;
+ optional bool merge_enabled = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/test.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/test.proto b/hbase-native-client/src/hbase/if/test.proto
new file mode 100644
index 0000000..72b68e9
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/test.proto
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
+option java_outer_classname = "TestProtos";
+option java_generate_equals_and_hash = true;
+
+message EmptyRequestProto {
+}
+
+message EmptyResponseProto {
+}
+
+message EchoRequestProto {
+ required string message = 1;
+}
+
+message EchoResponseProto {
+ required string message = 1;
+}
+
+message PauseRequestProto {
+ required uint32 ms = 1;
+}
+
+message AddrResponseProto {
+ required string addr = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/if/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/if/test_rpc_service.proto b/hbase-native-client/src/hbase/if/test_rpc_service.proto
new file mode 100644
index 0000000..2730403
--- /dev/null
+++ b/hbase-native-client/src/hbase/if/test_rpc_service.proto
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
+option java_outer_classname = "TestRpcServiceProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "test.proto";
+
+
+/**
+ * A protobuf service for use in tests
+ */
+service TestProtobufRpcProto {
+ rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc echo(EchoRequestProto) returns (EchoResponseProto);
+ rpc error(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc pause(PauseRequestProto) returns (EmptyResponseProto);
+ rpc addr(EmptyRequestProto) returns (AddrResponseProto);
+ rpc socketNotOpen(EmptyRequestProto) returns (EmptyResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/security/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/security/BUCK b/hbase-native-client/src/hbase/security/BUCK
new file mode 100644
index 0000000..c329f30
--- /dev/null
+++ b/hbase-native-client/src/hbase/security/BUCK
@@ -0,0 +1,27 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the library dealing with a single connection
+# to a single server.
+cxx_library(
+ name="security",
+ srcs=[],
+ deps=["//include/hbase/security:security", "//src/hbase/client:conf"],
+ compiler_flags=['-Weffc++'],
+ visibility=[
+ 'PUBLIC',
+ ],)
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/BUCK b/hbase-native-client/src/hbase/serde/BUCK
new file mode 100644
index 0000000..6b39e0b
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/BUCK
@@ -0,0 +1,86 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(
+ name="serde",
+ srcs=[
+ "rpc-serde.cc",
+ "zk.cc",
+ ],
+ deps=[
+ "//include/hbase/serde:serde", "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security"
+ ],
+ tests=[
+ ":client-deserializer-test",
+ ":client-serializer-test",
+ ":server-name-test",
+ ":table-name-test",
+ ":zk-deserializer-test",
+ ":region-info-deserializer-test",
+ ],
+ compiler_flags=['-Weffc++'],
+ visibility=[
+ 'PUBLIC',
+ ],)
+cxx_test(
+ name="table-name-test",
+ srcs=[
+ "table-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
+cxx_test(
+ name="server-name-test",
+ srcs=[
+ "server-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
+cxx_test(
+ name="client-serializer-test",
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
+cxx_test(
+ name="client-deserializer-test",
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
+cxx_test(
+ name="zk-deserializer-test",
+ srcs=[
+ "zk-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
+cxx_test(
+ name="region-info-deserializer-test",
+ srcs=[
+ "region-info-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/client-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/client-deserializer-test.cc b/hbase-native-client/src/hbase/serde/client-deserializer-test.cc
new file mode 100644
index 0000000..3e4c42b
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/client-deserializer-test.cc
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+
+#include "hbase/if/Client.pb.h"
+#include "hbase/serde/rpc-serde.h"
+
+using namespace hbase;
+using folly::IOBuf;
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+TEST(TestRpcSerde, TestReturnFalseOnNullPtr) {
+ RpcSerde deser{nullptr};
+ ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0);
+}
+
+TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
+ RpcSerde deser{nullptr};
+ auto buf = IOBuf::copyBuffer("test");
+ GetRequest gr;
+
+ ASSERT_LT(deser.ParseDelimited(buf.get(), &gr), 0);
+}
+
+TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
+ GetRequest in;
+ RpcSerde ser{nullptr};
+ RpcSerde deser{nullptr};
+
+ // fill up the GetRequest.
+ in.mutable_region()->set_value("test_region_id");
+ in.mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ in.mutable_get()->set_row("test_row");
+
+ // Create the buffer
+ auto buf = ser.SerializeDelimited(in);
+
+ GetRequest out;
+
+ int used_bytes = deser.ParseDelimited(buf.get(), &out);
+
+ ASSERT_GT(used_bytes, 0);
+ ASSERT_EQ(used_bytes, buf->length());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/client-serializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/client-serializer-test.cc b/hbase-native-client/src/hbase/serde/client-serializer-test.cc
new file mode 100644
index 0000000..8279caa
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/client-serializer-test.cc
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <gtest/gtest.h>
+
+#include <folly/io/Cursor.h>
+
+#include <string>
+
+#include "hbase/if/HBase.pb.h"
+#include "hbase/if/RPC.pb.h"
+#include "hbase/serde/rpc-serde.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace folly::io;
+
+TEST(RpcSerdeTest, PreambleIncludesHBas) {
+ RpcSerde ser{nullptr};
+ auto buf = ser.Preamble(false);
+ const char *p = reinterpret_cast<const char *>(buf->data());
+ // Take the first for chars and make sure they are the
+ // magic string
+ EXPECT_EQ("HBas", std::string(p, 4));
+
+ EXPECT_EQ(6, buf->computeChainDataLength());
+}
+
+TEST(RpcSerdeTest, PreambleIncludesVersion) {
+ RpcSerde ser{nullptr};
+ auto buf = ser.Preamble(false);
+ EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
+ EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
+}
+
+TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
+ RpcSerde ser{nullptr};
+ auto header = ser.Header("elliott");
+
+ // The header should be prefixed by 4 bytes of length.
+ EXPECT_EQ(4, header->length());
+ EXPECT_TRUE(header->length() < header->computeChainDataLength());
+ EXPECT_TRUE(header->isChained());
+
+ // Now make sure the length is correct.
+ Cursor cursor(header.get());
+ auto prefixed_len = cursor.readBE<uint32_t>();
+ EXPECT_EQ(prefixed_len, header->next()->length());
+}
+
+TEST(RpcSerdeTest, TestHeaderDecode) {
+ RpcSerde ser{nullptr};
+ auto buf = ser.Header("elliott");
+ auto header_buf = buf->next();
+ ConnectionHeader h;
+
+ EXPECT_TRUE(h.ParseFromArray(header_buf->data(), header_buf->length()));
+ EXPECT_EQ("elliott", h.user_info().effective_user());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc b/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc
new file mode 100644
index 0000000..84219d8
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/region-info-deserializer-test.cc
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/serde/region-info.h"
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/table-name.h"
+
+using std::string;
+using hbase::pb::RegionInfo;
+using hbase::pb::TableName;
+
+TEST(TestRegionInfoDesializer, TestDeserialize) {
+ string ns{"test_ns"};
+ string tn{"table_name"};
+ string start_row{"AAAAAA"};
+ string stop_row{"BBBBBBBBBBBB"};
+ uint64_t region_id = 2345678;
+
+ RegionInfo ri_out;
+ ri_out.set_region_id(region_id);
+ ri_out.mutable_table_name()->set_namespace_(ns);
+ ri_out.mutable_table_name()->set_qualifier(tn);
+ ri_out.set_start_key(start_row);
+ ri_out.set_end_key(stop_row);
+
+ string header{"PBUF"};
+ string ser = header + ri_out.SerializeAsString();
+
+ auto out = folly::to<RegionInfo>(ser);
+
+ EXPECT_EQ(region_id, out.region_id());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/rpc-serde.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/rpc-serde.cc b/hbase-native-client/src/hbase/serde/rpc-serde.cc
new file mode 100644
index 0000000..5f2920d
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/rpc-serde.cc
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/message.h>
+#include <boost/algorithm/string.hpp>
+
+#include <utility>
+
+#include "hbase/if/RPC.pb.h"
+#include "hbase/serde/rpc-serde.h"
+#include "hbase/utils/version.h"
+
+using folly::IOBuf;
+using folly::io::RWPrivateCursor;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayInputStream;
+using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedInputStream;
+using google::protobuf::io::CodedOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
+
+using namespace hbase::pb;
+
+namespace hbase {
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t DEFAULT_AUTH_TYPE = 80;
+static const uint8_t KERBEROS_AUTH_TYPE = 81;
+
+int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
+ if (buf == nullptr || msg == nullptr) {
+ return -2;
+ }
+
+ DCHECK(!buf->isChained());
+
+ ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
+ CodedInputStream coded_stream{&ais};
+
+ uint32_t msg_size;
+
+ // Try and read the varint.
+ if (coded_stream.ReadVarint32(&msg_size) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
+ return -3;
+ }
+
+ coded_stream.PushLimit(msg_size);
+ // Parse the message.
+ if (msg->MergeFromCodedStream(&coded_stream) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a protobuf message from data.";
+ return -4;
+ }
+
+ // Make sure all the data was consumed.
+ if (coded_stream.ConsumedEntireMessage() == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Orphaned data left after reading protobuf message";
+ return -5;
+ }
+
+ return coded_stream.CurrentPosition();
+}
+
+RpcSerde::RpcSerde() {}
+
+RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : codec_(codec) {}
+
+std::unique_ptr<IOBuf> RpcSerde::Preamble(bool secure) {
+ auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
+ magic->append(2);
+ RWPrivateCursor c(magic.get());
+ c.skip(4);
+ // Version
+ c.write(RPC_VERSION);
+ if (secure) {
+ // for now support only KERBEROS (DIGEST is not supported)
+ c.write(KERBEROS_AUTH_TYPE);
+ } else {
+ c.write(DEFAULT_AUTH_TYPE);
+ }
+ return magic;
+}
+
+std::unique_ptr<IOBuf> RpcSerde::Header(const std::string &user) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user(user);
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ // TODO: send the service name and user from the RpcClient
+ h.set_service_name(INTERFACE);
+
+ std::unique_ptr<pb::VersionInfo> version_info = CreateVersionInfo();
+
+ h.set_allocated_version_info(version_info.release());
+
+ if (codec_ != nullptr) {
+ h.set_cell_block_codec_class(codec_->java_class_name());
+ }
+ return PrependLength(SerializeMessage(h));
+}
+
+std::unique_ptr<pb::VersionInfo> RpcSerde::CreateVersionInfo() {
+ std::unique_ptr<pb::VersionInfo> version_info = std::make_unique<pb::VersionInfo>();
+ version_info->set_user(Version::user);
+ version_info->set_revision(Version::revision);
+ version_info->set_url(Version::url);
+ version_info->set_date(Version::date);
+ version_info->set_src_checksum(Version::src_checksum);
+ version_info->set_version(Version::version);
+
+ std::string version{Version::version};
+ std::vector<std::string> version_parts;
+ boost::split(version_parts, version, boost::is_any_of("."), boost::token_compress_on);
+ uint32_t major_version = 0, minor_version = 0;
+ if (version_parts.size() >= 2) {
+ version_info->set_version_major(folly::to<uint32_t>(version_parts[0]));
+ version_info->set_version_minor(folly::to<uint32_t>(version_parts[1]));
+ }
+
+ VLOG(1) << "Client VersionInfo:" << version_info->ShortDebugString();
+ return version_info;
+}
+
+std::unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const std::string &method,
+ const Message *msg) {
+ pb::RequestHeader rq;
+ rq.set_method_name(method);
+ rq.set_call_id(call_id);
+ rq.set_request_param(msg != nullptr);
+ auto ser_header = SerializeDelimited(rq);
+ if (msg != nullptr) {
+ auto ser_req = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_req));
+ }
+
+ return PrependLength(std::move(ser_header));
+}
+
+std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id,
+ const google::protobuf::Message *msg) {
+ pb::ResponseHeader rh;
+ rh.set_call_id(call_id);
+ auto ser_header = SerializeDelimited(rh);
+ auto ser_resp = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_resp));
+
+ return PrependLength(std::move(ser_header));
+}
+
+std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id,
+ const google::protobuf::Message *msg,
+ const folly::exception_wrapper &exception) {
+ /* create ResponseHeader */
+ pb::ResponseHeader rh;
+ rh.set_call_id(call_id);
+
+ /* create ExceptionResponse */
+ if (bool(exception)) {
+ VLOG(1) << "packing ExceptionResponse";
+ auto exception_response = new pb::ExceptionResponse();
+ exception_response->set_exception_class_name(exception.class_name().c_str());
+ exception_response->set_stack_trace(exception.what().c_str());
+ rh.set_allocated_exception(exception_response);
+ }
+
+ /* serialize Response header and body */
+ auto ser_header = SerializeDelimited(rh);
+ auto ser_resp = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_resp));
+
+ VLOG(3) << "Converted hbase::Response to folly::IOBuf";
+ return PrependLength(std::move(ser_header));
+}
+
+std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::IOBuf> buf,
+ uint32_t offset, uint32_t length) {
+ if (codec_ == nullptr) {
+ return nullptr;
+ }
+ return codec_->CreateDecoder(std::move(buf), offset, length);
+}
+
+std::unique_ptr<IOBuf> RpcSerde::PrependLength(std::unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
+
+std::unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
+ // Get the buffer size needed for just the message.
+ int msg_size = msg.ByteSize();
+ int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
+
+ // Create a buffer big enough to hold the varint and the object.
+ auto buf = IOBuf::create(buf_size);
+ buf->append(buf_size);
+
+ // Create the array output stream.
+ ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
+ // Wrap the ArrayOuputStream in the coded output stream to allow writing
+ // Varint32
+ CodedOutputStream cos{&aos};
+
+ // Write out the size.
+ cos.WriteVarint32(msg_size);
+
+ // Now write the rest out.
+ // We're using the protobuf output streams here to keep track
+ // of where in the output array we are rather than IOBuf.
+ msg.SerializeWithCachedSizesToArray(cos.GetDirectBufferForNBytesAndAdvance(msg_size));
+
+ // Return the buffer.
+ return buf;
+}
+// TODO(eclark): Make this 1 copy.
+std::unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
+ auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
+ return buf;
+}
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/server-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/server-name-test.cc b/hbase-native-client/src/hbase/serde/server-name-test.cc
new file mode 100644
index 0000000..12d3287
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/server-name-test.cc
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/serde/server-name.h"
+
+#include <gtest/gtest.h>
+#include <string>
+
+using hbase::pb::ServerName;
+
+TEST(TestServerName, TestMakeServerName) {
+ auto sn = folly::to<ServerName>("test:123");
+
+ ASSERT_EQ("test", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+}
+
+TEST(TestServerName, TestIps) {
+ auto sn = folly::to<ServerName>("127.0.0.1:999");
+ ASSERT_EQ("127.0.0.1", sn.host_name());
+ ASSERT_EQ(999, sn.port());
+}
+
+TEST(TestServerName, TestThrow) { ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here")); }
+
+TEST(TestServerName, TestIPV6) {
+ auto sn = folly::to<ServerName>("[::::1]:123");
+
+ ASSERT_EQ("[::::1]", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/table-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/table-name-test.cc b/hbase-native-client/src/hbase/serde/table-name-test.cc
new file mode 100644
index 0000000..77bd6c2
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/table-name-test.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "hbase/serde/table-name.h"
+
+using namespace hbase;
+using hbase::pb::TableName;
+
+TEST(TestTableName, TestToStringNoDefault) {
+ TableName tn;
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
+TEST(TestTableName, TestToStringNoDefaltWhenSet) {
+ TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
+TEST(TestTableName, TestToStringIncludeNS) {
+ TableName tn;
+ tn.set_namespace_("hbase");
+ tn.set_qualifier("acl");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("hbase"), 0);
+ ASSERT_EQ("hbase:acl", result);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc b/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc
new file mode 100644
index 0000000..141efce
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/zk-deserializer-test.cc
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/serde/zk.h"
+
+#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+
+#include "hbase/if/ZooKeeper.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace std;
+using namespace folly::io;
+
+// Test that would test if there's nothing there.
+TEST(TestZkDesializer, TestThrowNoMagicNum) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+
+ auto buf = IOBuf::create(100);
+ buf->append(100);
+ RWPrivateCursor c{buf.get()};
+ c.write<uint8_t>(99);
+ ASSERT_THROW(deser.Parse(buf.get(), &mrs), runtime_error);
+}
+
+// Test if the protobuf is in a format that we can't decode
+TEST(TestZkDesializer, TestBadProtoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_THROW(deser.Parse(buf.get(), &out), runtime_error);
+}
+
+// Test to make sure the whole thing works.
+TEST(TestZkDesializer, TestNoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Now write the serialized protobuf
+ mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_TRUE(deser.Parse(buf.get(), &out));
+ ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/serde/zk.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/serde/zk.cc b/hbase-native-client/src/hbase/serde/zk.cc
new file mode 100644
index 0000000..b962cc5
--- /dev/null
+++ b/hbase-native-client/src/hbase/serde/zk.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/serde/zk.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <google/protobuf/message.h>
+
+#include <string>
+
+using std::runtime_error;
+
+namespace hbase {
+
+static const std::string MAGIC_STRING = "PBUF";
+
+bool ZkDeserializer::Parse(folly::IOBuf *buf, google::protobuf::Message *out) {
+ // The format is like this
+ // 1 byte of magic number. 255
+ // 4 bytes of id length.
+ // id_length number of bytes for the id of who put up the znode
+ // 4 bytes of a magic string PBUF
+ // Then the protobuf serialized without a varint header.
+
+ folly::io::Cursor c{buf};
+
+ // There should be a magic number for recoverable zk
+ uint8_t magic_num = c.read<uint8_t>();
+ if (magic_num != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" << unsigned(magic_num);
+ throw runtime_error("Magic number not in znode data");
+ }
+ // How long is the id?
+ uint32_t id_len = c.readBE<uint32_t>();
+
+ if (id_len >= c.length()) {
+ LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
+ "left to read anything else";
+ throw runtime_error("Not enough bytes to decode from zookeeper");
+ }
+
+ // Skip the id
+ c.skip(id_len);
+
+ // Make sure that the magic string is there.
+ if (MAGIC_STRING != c.readFixedString(4)) {
+ LOG(ERROR) << "There was no PBUF magic string.";
+ throw runtime_error("No PBUF magic string in the zookpeeper data.");
+ }
+
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (out->ParseFromArray(c.data(), c.length()) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
+ return true;
+}
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/test-util/BUCK b/hbase-native-client/src/hbase/test-util/BUCK
new file mode 100644
index 0000000..f1aedab
--- /dev/null
+++ b/hbase-native-client/src/hbase/test-util/BUCK
@@ -0,0 +1,53 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+cxx_library(
+ name="test-util",
+ header_namespace="hbase/test-util",
+ exported_headers=["test-util.h", "mini-cluster.h"],
+ srcs=["test-util.cc", "mini-cluster.cc"],
+ deps=[
+ "//third-party:folly",
+ "//src/hbase/client:client",
+ ],
+ preprocessor_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'
+ ],
+ exported_preprocessor_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'
+ ],
+ compiler_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'
+ ],
+ linker_flags=[
+ '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
+ '-ggdb'
+ ],
+ exported_linker_flags=[
+ '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
+ '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],)
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/mini-cluster.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/test-util/mini-cluster.cc b/hbase-native-client/src/hbase/test-util/mini-cluster.cc
new file mode 100644
index 0000000..1e491a2
--- /dev/null
+++ b/hbase-native-client/src/hbase/test-util/mini-cluster.cc
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/test-util/mini-cluster.h"
+#include <fcntl.h>
+#include <glog/logging.h>
+#include <boost/filesystem/fstream.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <fstream>
+
+using hbase::MiniCluster;
+
+JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
+ JavaVMInitArgs args;
+ JavaVMOption jvm_options;
+ args.version = JNI_VERSION_1_6;
+ args.nOptions = 1;
+ char *classpath = getenv("CLASSPATH");
+ std::string clspath;
+ if (classpath == NULL || strstr(classpath, "-tests.jar") == NULL) {
+ std::string clsPathFilePath("../target/cached_classpath.txt");
+ std::ifstream fd(clsPathFilePath);
+ std::string prefix("");
+ if (fd.is_open()) {
+ if (classpath == NULL) {
+ LOG(INFO) << "got empty classpath";
+ } else {
+ // prefix bootstrapper.jar
+ prefix.assign(classpath);
+ }
+ std::string line;
+ if (getline(fd, line)) {
+ clspath = prefix + ":" + line;
+ int ret = setenv("CLASSPATH", clspath.c_str(), 1);
+ LOG(INFO) << "set clspath " << ret;
+ } else {
+ LOG(INFO) << "nothing read from " << clsPathFilePath;
+ exit(-1);
+ }
+ } else {
+ LOG(INFO) << "nothing read from " << clsPathFilePath;
+ exit(-1);
+ }
+ fd.close();
+ }
+
+ auto options = std::string{"-Djava.class.path="} + clspath;
+ jvm_options.optionString = const_cast<char *>(options.c_str());
+ args.options = &jvm_options;
+ args.ignoreUnrecognized = 0;
+ int rv;
+ rv = JNI_CreateJavaVM(jvm, reinterpret_cast<void **>(&env_), &args);
+ CHECK(rv >= 0 && env_);
+ return env_;
+}
+
+MiniCluster::~MiniCluster() {
+ if (jvm_ != NULL) {
+ jvm_->DestroyJavaVM();
+ jvm_ = NULL;
+ }
+ env_ = nullptr;
+}
+
+void MiniCluster::Setup() {
+ jmethodID constructor;
+ pthread_mutex_lock(&count_mutex_);
+ if (env_ == NULL) {
+ env_ = CreateVM(&jvm_);
+ if (env_ == NULL) {
+ exit(-1);
+ }
+ testing_util_class_ = env_->FindClass("org/apache/hadoop/hbase/HBaseTestingUtility");
+ // this should be converted to a globalref I think to avoid the underlying java obj getting
+ // GC'ed
+ if (testing_util_class_ == NULL) {
+ LOG(INFO) << "Couldn't find class HBaseTestingUtility";
+ exit(-1);
+ }
+ jmethodID mid = env_->GetStaticMethodID(testing_util_class_, "createLocalHTU",
+ "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;");
+ htu_ = env_->CallStaticObjectMethod(testing_util_class_, mid);
+ // this should be converted to a globalref I think to avoid the underlying java obj getting
+ // GC'ed
+ if (htu_ == NULL) {
+ LOG(INFO) << "Couldn't invoke method createLocalHTU in HBaseTestingUtility";
+ exit(-1);
+ }
+ get_conn_mid_ = env_->GetMethodID(testing_util_class_, "getConnection",
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
+ jclass conn_class = env_->FindClass("org/apache/hadoop/hbase/client/Connection");
+ get_admin_mid_ =
+ env_->GetMethodID(conn_class, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;");
+ get_table_mid_ = env_->GetMethodID(
+ conn_class, "getTable",
+ "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;");
+ if (get_table_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find getConnection";
+ exit(-1);
+ }
+ jclass adminClass = env_->FindClass("org/apache/hadoop/hbase/client/Admin");
+ move_mid_ = env_->GetMethodID(adminClass, "move", "([B[B)V");
+ if (move_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find move";
+ exit(-1);
+ }
+ create_table_mid_ =
+ env_->GetMethodID(testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/"
+ "apache/hadoop/hbase/client/Table;");
+ create_table_families_mid_ = env_->GetMethodID(testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;[[B)Lorg/"
+ "apache/hadoop/hbase/client/Table;");
+ create_table_with_split_mid_ = env_->GetMethodID(
+ testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;");
+ if (create_table_with_split_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method createTable with split";
+ exit(-1);
+ }
+
+ table_name_class_ = env_->FindClass("org/apache/hadoop/hbase/TableName");
+ tbl_name_value_of_mid_ = env_->GetStaticMethodID(
+ table_name_class_, "valueOf", "(Ljava/lang/String;)Lorg/apache/hadoop/hbase/TableName;");
+ if (tbl_name_value_of_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method valueOf in TableName";
+ exit(-1);
+ }
+ jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster");
+ stop_rs_mid_ =
+ env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer",
+ "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;");
+ get_conf_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration",
+ "()Lorg/apache/hadoop/conf/Configuration;");
+
+ conf_class_ = env_->FindClass("org/apache/hadoop/conf/Configuration");
+ set_conf_mid_ =
+ env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V");
+ if (set_conf_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster";
+ exit(-1);
+ }
+ conf_get_mid_ = env_->GetMethodID(conf_class_, "get", "(Ljava/lang/String;)Ljava/lang/String;");
+
+ jclass tableClass = env_->FindClass("org/apache/hadoop/hbase/client/Table");
+ put_mid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V");
+ jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory");
+ create_conn_mid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection",
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
+ if (create_conn_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find createConnection";
+ exit(-1);
+ }
+ put_class_ = env_->FindClass("org/apache/hadoop/hbase/client/Put");
+ put_ctor_ = env_->GetMethodID(put_class_, "<init>", "([B)V");
+ add_col_mid_ =
+ env_->GetMethodID(put_class_, "addColumn", "([B[B[B)Lorg/apache/hadoop/hbase/client/Put;");
+ if (add_col_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method addColumn";
+ exit(-1);
+ }
+ }
+ pthread_mutex_unlock(&count_mutex_);
+}
+
+jobject MiniCluster::htu() {
+ Setup();
+ return htu_;
+}
+
+JNIEnv *MiniCluster::env() {
+ Setup();
+ return env_;
+}
+// converts C char* to Java byte[]
+jbyteArray MiniCluster::StrToByteChar(const std::string &str) {
+ if (str.length() == 0) {
+ return nullptr;
+ }
+ int n = str.length();
+ jbyteArray arr = env_->NewByteArray(n);
+ env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast<const jbyte *>(str.c_str()));
+ return arr;
+}
+
+jobject MiniCluster::CreateTable(const std::string &table, const std::string &family) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jstring family_str = env_->NewStringUTF(family.c_str());
+ jobject table_obj = env_->CallObjectMethod(htu_, create_table_mid_, table_name, family_str);
+ return table_obj;
+}
+
+jobject MiniCluster::CreateTable(const std::string &table,
+ const std::vector<std::string> &families) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jclass array_element_type = env_->FindClass("[B");
+ jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr);
+ int i = 0;
+ for (auto family : families) {
+ env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family));
+ }
+ jobject table_obj =
+ env_->CallObjectMethod(htu_, create_table_families_mid_, table_name, family_array);
+ return table_obj;
+}
+
+jobject MiniCluster::CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys) {
+ std::vector<std::string> families{};
+ families.push_back(std::string{family});
+ return CreateTable(table, families, keys);
+}
+
+jobject MiniCluster::CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jclass array_element_type = env_->FindClass("[B");
+
+ int i = 0;
+ jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr);
+ for (auto family : families) {
+ env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family));
+ }
+
+ jobjectArray key_array = env_->NewObjectArray(keys.size(), array_element_type, nullptr);
+
+ i = 0;
+ for (auto key : keys) {
+ env_->SetObjectArrayElement(key_array, i++, StrToByteChar(key));
+ }
+
+ jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, table_name, family_array,
+ key_array);
+ return tbl;
+}
+
+jobject MiniCluster::StopRegionServer(int idx) {
+ env();
+ return env_->CallObjectMethod(cluster_, stop_rs_mid_, (jint)idx);
+}
+
+// returns the Configuration for the cluster
+jobject MiniCluster::GetConf() {
+ env();
+ return env_->CallObjectMethod(cluster_, get_conf_mid_);
+}
+// return the Admin instance for the local cluster
+jobject MiniCluster::admin() {
+ env();
+ jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_);
+ jobject admin = env_->CallObjectMethod(conn, get_admin_mid_);
+ return admin;
+}
+
+// moves region to server
+void MiniCluster::MoveRegion(const std::string ®ion, const std::string &server) {
+ jobject admin_ = admin();
+ env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region), StrToByteChar(server));
+}
+
+jobject MiniCluster::StartCluster(int num_region_servers) {
+ env();
+ jmethodID mid = env_->GetMethodID(testing_util_class_, "startMiniCluster",
+ "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;");
+ if (mid == NULL) {
+ LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility";
+ exit(-1);
+ }
+ cluster_ = env_->CallObjectMethod(htu(), mid, static_cast<jint>(num_region_servers));
+ return cluster_;
+}
+
+void MiniCluster::StopCluster() {
+ env();
+ jmethodID mid = env_->GetMethodID(testing_util_class_, "shutdownMiniCluster", "()V");
+ env_->CallVoidMethod(htu(), mid);
+ if (jvm_ != NULL) {
+ jvm_->DestroyJavaVM();
+ jvm_ = NULL;
+ }
+}
+
+const std::string MiniCluster::GetConfValue(const std::string &key) {
+ jobject conf = GetConf();
+ jstring jval =
+ (jstring)env_->CallObjectMethod(conf, conf_get_mid_, env_->NewStringUTF(key.c_str()));
+ const char *val = env_->GetStringUTFChars(jval, 0);
+ return val;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/mini-cluster.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/test-util/mini-cluster.h b/hbase-native-client/src/hbase/test-util/mini-cluster.h
new file mode 100644
index 0000000..6b4547c
--- /dev/null
+++ b/hbase-native-client/src/hbase/test-util/mini-cluster.h
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <jni.h>
+#include <string>
+#include <vector>
+
+namespace hbase {
+
+class MiniCluster {
+ public:
+ virtual ~MiniCluster();
+ jobject StartCluster(int32_t num_region_servers);
+ void StopCluster();
+ jobject CreateTable(const std::string &table, const std::string &family);
+ jobject CreateTable(const std::string &table, const std::vector<std::string> &families);
+ jobject CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys);
+ jobject CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys);
+ jobject StopRegionServer(int idx);
+
+ // moves region to server
+ void MoveRegion(const std::string ®ion, const std::string &server);
+ // returns the Configuration instance for the cluster
+ jobject GetConf();
+ // returns the value for config key retrieved from cluster
+ const std::string GetConfValue(const std::string &key);
+
+ private:
+ JNIEnv *env_;
+ jclass testing_util_class_;
+ jclass table_name_class_;
+ jclass put_class_;
+ jclass conf_class_;
+ jmethodID stop_rs_mid_;
+ jmethodID get_conf_mid_;
+ jmethodID set_conf_mid_;
+ jmethodID tbl_name_value_of_mid_;
+ jmethodID create_table_mid_;
+ jmethodID create_table_families_mid_;
+ jmethodID create_table_with_split_mid_;
+ jmethodID put_mid_;
+ jmethodID put_ctor_;
+ jmethodID add_col_mid_;
+ jmethodID create_conn_mid_;
+ jmethodID get_conn_mid_;
+ jmethodID get_table_mid_;
+ jmethodID conf_get_mid_;
+ jmethodID get_admin_mid_;
+ jmethodID move_mid_;
+ jmethodID str_ctor_mid_;
+ jobject htu_;
+ jobject cluster_;
+ pthread_mutex_t count_mutex_;
+ JavaVM *jvm_;
+ JNIEnv *CreateVM(JavaVM **jvm);
+ void Setup();
+ jobject htu();
+ JNIEnv *env();
+ jbyteArray StrToByteChar(const std::string &str);
+ jobject admin();
+};
+} /*namespace hbase*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/test-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/test-util/test-util.cc b/hbase-native-client/src/hbase/test-util/test-util.cc
new file mode 100644
index 0000000..ebaf701
--- /dev/null
+++ b/hbase-native-client/src/hbase/test-util/test-util.cc
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "hbase/test-util/test-util.h"
+#include <string.h>
+
+#include <folly/Format.h>
+
+#include "hbase/client/zk-util.h"
+
+using hbase::TestUtil;
+using folly::Random;
+
+std::string TestUtil::RandString(int len) {
+ // Create the whole string.
+ // Filling everything with z's
+ auto s = std::string(len, 'z');
+
+ // Now pick a bunch of random numbers
+ for (int i = 0; i < len; i++) {
+ // use Folly's random to get the numbers
+ // as I don't want to have to learn
+ // all the cpp rand invocation magic.
+ auto r = Random::rand32('a', 'z');
+ // Cast that to ascii.
+ s[i] = static_cast<char>(r);
+ }
+ return s;
+}
+
+TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {}
+
+TestUtil::~TestUtil() {
+ if (mini_) {
+ StopMiniCluster();
+ mini_ = nullptr;
+ }
+}
+
+void TestUtil::StartMiniCluster(int32_t num_region_servers) {
+ mini_ = std::make_unique<MiniCluster>();
+ mini_->StartCluster(num_region_servers);
+
+ conf()->Set(ZKUtil::kHBaseZookeeperQuorum_, mini_->GetConfValue(ZKUtil::kHBaseZookeeperQuorum_));
+ conf()->Set(ZKUtil::kHBaseZookeeperClientPort_,
+ mini_->GetConfValue(ZKUtil::kHBaseZookeeperClientPort_));
+}
+
+void TestUtil::StopMiniCluster() { mini_->StopCluster(); }
+
+void TestUtil::CreateTable(const std::string &table, const std::string &family) {
+ mini_->CreateTable(table, family);
+}
+
+void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families) {
+ mini_->CreateTable(table, families);
+}
+
+void TestUtil::CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys) {
+ mini_->CreateTable(table, family, keys);
+}
+
+void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys) {
+ mini_->CreateTable(table, families, keys);
+}
+
+void TestUtil::MoveRegion(const std::string ®ion, const std::string &server) {
+ mini_->MoveRegion(region, server);
+}
+
+void TestUtil::StartStandAloneInstance() {
+ auto p = temp_dir_.path().string();
+ auto cmd = std::string{"bin/start-local-hbase.sh " + p};
+ auto res_code = std::system(cmd.c_str());
+ CHECK_EQ(res_code, 0);
+}
+
+void TestUtil::StopStandAloneInstance() {
+ auto res_code = std::system("bin/stop-local-hbase.sh");
+ CHECK_EQ(res_code, 0);
+}
+
+void TestUtil::RunShellCmd(const std::string &command) {
+ auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
+ auto res_code = std::system(cmd_string.c_str());
+ CHECK_EQ(res_code, 0);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/test-util/test-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/test-util/test-util.h b/hbase-native-client/src/hbase/test-util/test-util.h
new file mode 100644
index 0000000..7c57c28
--- /dev/null
+++ b/hbase-native-client/src/hbase/test-util/test-util.h
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Random.h>
+#include <folly/experimental/TestUtil.h>
+
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <vector>
+#include "hbase/client/configuration.h"
+#include "hbase/test-util/mini-cluster.h"
+
+namespace hbase {
+/**
+ * @brief Class to deal with a local instance cluster for testing.
+ */
+class TestUtil {
+ public:
+ TestUtil();
+
+ /**
+ * Destroying a TestUtil will spin down a cluster and remove the test dir.
+ */
+ ~TestUtil();
+
+ /**
+ * Create a random string. This random string is all letters, as such it is
+ * very good for use as a directory name.
+ */
+ static std::string RandString(int len = 32);
+
+ /**
+ * Returns the configuration to talk to the local cluster
+ */
+ std::shared_ptr<Configuration> conf() const { return conf_; }
+
+ /**
+ * Starts mini hbase cluster with specified number of region servers
+ */
+ void StartMiniCluster(int32_t num_region_servers);
+
+ void StopMiniCluster();
+ void CreateTable(const std::string &table, const std::string &family);
+ void CreateTable(const std::string &table, const std::vector<std::string> &families);
+ void CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys);
+ void CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys);
+
+ void StartStandAloneInstance();
+ void StopStandAloneInstance();
+ void RunShellCmd(const std::string &);
+ void MoveRegion(const std::string ®ion, const std::string &server);
+
+ private:
+ std::unique_ptr<MiniCluster> mini_;
+ folly::test::TemporaryDirectory temp_dir_;
+ std::shared_ptr<Configuration> conf_ = std::make_shared<Configuration>();
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/utils/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/utils/BUCK b/hbase-native-client/src/hbase/utils/BUCK
new file mode 100644
index 0000000..ab55d8f
--- /dev/null
+++ b/hbase-native-client/src/hbase/utils/BUCK
@@ -0,0 +1,57 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(
+ name="utils",
+ exported_headers=[
+ ],
+ srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
+ deps=[
+ '//include/hbase/utils:utils',
+ '//third-party:folly',
+ ],
+ tests=[":user-util-test"],
+ linker_flags=['-L/usr/local/lib', '-lkrb5'],
+ exported_linker_flags=['-L/usr/local/lib', '-lkrb5'],
+ visibility=[
+ 'PUBLIC',
+ ],
+ compiler_flags=['-Weffc++'],)
+cxx_test(
+ name="bytes-util-test",
+ srcs=[
+ "bytes-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
+cxx_test(
+ name="concurrent-map-test",
+ srcs=[
+ "concurrent-map-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
+cxx_test(
+ name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
http://git-wip-us.apache.org/repos/asf/hbase/blob/128fc306/hbase-native-client/src/hbase/utils/bytes-util-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/hbase/utils/bytes-util-test.cc b/hbase-native-client/src/hbase/utils/bytes-util-test.cc
new file mode 100644
index 0000000..e601d2d
--- /dev/null
+++ b/hbase-native-client/src/hbase/utils/bytes-util-test.cc
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <gtest/gtest.h>
+#include <string>
+
+#include "hbase/utils/bytes-util.h"
+
+using hbase::BytesUtil;
+
+TEST(TestBytesUtil, TestToStringBinary) {
+ std::string empty{""};
+ EXPECT_EQ(empty, BytesUtil::ToStringBinary(empty));
+
+ std::string foo_bar{"foo bar"};
+ EXPECT_EQ(foo_bar, BytesUtil::ToStringBinary(foo_bar));
+
+ std::string foo_bar2{"foo bar_/!@#$%^&*(){}[]|1234567890"};
+ EXPECT_EQ(foo_bar2, BytesUtil::ToStringBinary(foo_bar2));
+
+ char zero = 0;
+ EXPECT_EQ("\\x00", BytesUtil::ToStringBinary(std::string{zero}));
+
+ char max = 255;
+ EXPECT_EQ("\\xFF", BytesUtil::ToStringBinary(std::string{max}));
+
+ EXPECT_EQ("\\x00\\xFF", BytesUtil::ToStringBinary(std::string{zero} + std::string{max}));
+
+ EXPECT_EQ("foo_\\x00\\xFF_bar",
+ BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar"));
+}
+
+TEST(TestBytesUtil, TestToStringToInt64) {
+ int64_t num = 761235;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
+
+ num = -56125;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
+
+ num = 0;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
+}
+
+TEST(TestBytesUtil, TestCreateClosestRowAfter) {
+ std::string empty{""};
+ EXPECT_EQ(BytesUtil::CreateClosestRowAfter(empty), std::string{'\0'});
+
+ std::string foo{"foo"};
+ EXPECT_EQ(BytesUtil::CreateClosestRowAfter(foo), std::string{"foo"} + '\0');
+
+ EXPECT_EQ("f\\x00", BytesUtil::ToStringBinary(BytesUtil::CreateClosestRowAfter("f")));
+}